Revert "lib/model: Add folders on start in model (#6135)"

This reverts commit bee7cce081.

.golangci.yml

@@ -12,9 +12,13 @@ linters:
     - gochecknoglobals
     - gofmt
     - scopelint
+    - gocyclo
+    - funlen
+    - wsl
 
 service:
-  golangci-lint-version: 1.16.x
+  golangci-lint-version: 1.21.x
   prepare:
+    - rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
     - GO111MODULE=on go mod vendor
     - go run build.go assets

AUTHORS

@@ -46,7 +46,7 @@ Brandon Philips (philips) <brandon@ifup.org>
 Brendan Long (brendanlong) <self@brendanlong.com>
 Brian R. Becker (brbecker) <brbecker@gmail.com>
 Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
-Carsten Hagemann (Moter8) <moter8@gmail.com>
+Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
 Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
 Cedric Staniewski (xduugu) <cedric@gmx.ca>
 Chris Howie (cdhowie) <me@chrishowie.com>

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.12 AS builder
+FROM golang:1.13 AS builder
 
 WORKDIR /src
 COPY . .
@@ -19,9 +19,10 @@ RUN apk add --no-cache ca-certificates su-exec
 COPY --from=builder /src/syncthing /bin/syncthing
 COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
 
-ENV PUID=1000 PGID=1000
+ENV PUID=1000 PGID=1000 HOME=/var/syncthing
 
 HEALTHCHECK --interval=1m --timeout=10s \
   CMD nc -z localhost 8384 || exit 1
 
-ENTRYPOINT ["/bin/entrypoint.sh", "-home", "/var/syncthing/config", "-gui-address", "0.0.0.0:8384"]
+ENV STGUIADDRESS=0.0.0.0:8384
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]

Dockerfile.stdiscosrv

@@ -0,0 +1,28 @@
+FROM golang:1.13 AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f stdiscosrv && go run build.go -no-upgrade build stdiscosrv
+
+FROM alpine
+
+EXPOSE 19200 8443
+
+VOLUME ["/var/stdiscosrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/stdiscosrv /bin/stdiscosrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/stdiscosrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 8443 || exit 1
+
+WORKDIR /var/stdiscosrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/stdiscosrv"]

Dockerfile.strelaysrv

@@ -0,0 +1,28 @@
+FROM golang:1.13 AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f strelaysrv && go run build.go -no-upgrade build strelaysrv
+
+FROM alpine
+
+EXPOSE 22067 22070
+
+VOLUME ["/var/strelaysrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/strelaysrv /bin/strelaysrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/strelaysrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 22067 || exit 1
+
+WORKDIR /var/strelaysrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaysrv"]

README-Docker.md

@@ -18,7 +18,11 @@ $ docker run -p 8384:8384 -p 22000:22000 \
     syncthing/syncthing:latest
 ```
 
-Note that local device discovery will not work with the above command, resulting in poor local transfer rates if local device addresses are not manually configured.
+## Discovery
+
+Note that local device discovery will not work with the above command,
+resulting in poor local transfer rates if local device addresses are not
+manually configured.
 
 To allow local discovery, the docker host network can be used instead:
 
@@ -32,3 +36,24 @@ $ docker run --network=host \
 Be aware that syncthing alone is now in control of what interfaces and ports it
 listens on. You can edit the syncthing configuration to change the defaults if
 there are conflicts.
+
+## GUI Security
+
+By default Syncthing inside the Docker image listens on 0.0.0.0:8384 to
+allow GUI connections via the Docker proxy. This is set by the
+`STGUIADDRESS` environment variable in the Dockerfile, as it differs from
+what Syncthing would otherwise use by default. This means you should set up
+authentication in the GUI, like for any other externally reachable Syncthing
+instance. If you do not require the GUI, or you use host networking, you can
+unset the `STGUIADDRESS` variable to have Syncthing fall back to listening
+on 127.0.0.1:
+
+```
+$ docker pull syncthing/syncthing
+$ docker run -e STGUIADDRESS= \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+With the environment variable unset Syncthing will follow what is set in the
+configuration file / GUI settings dialog.

README.md

@@ -62,6 +62,10 @@ There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory][3]. There are also several [GUI
 implementations][11] for Windows, Mac and Linux.
 
+## Docker
+
+To run Syncthing in Docker, see [the Docker README][16].
+
 ## Vote on features/bugs
 
 We'd like to encourage you to [vote][12] on issues that matter to you.
@@ -110,4 +114,5 @@ All code is licensed under the [MPLv2 License][7].
 [13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
 [14]: assets/logo-text-128.png
 [15]: https://syncthing.net/
+[16]: https://github.com/syncthing/syncthing/blob/master/README-Docker.md
 

build.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"os/exec"
 	"os/user"
+	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -34,34 +35,35 @@ import (
 )
 
 var (
-	versionRe        = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
-	goarch           string
-	goos             string
-	noupgrade        bool
-	version          string
-	goCmd            string
-	goVersion        float64
-	race             bool
-	debug            = os.Getenv("BUILDDEBUG") != ""
-	extraTags        string
-	installSuffix    string
-	pkgdir           string
-	cc               string
-	debugBinary      bool
-	coverage         bool
-	timeout          = "120s"
-	gogoProtoVersion = "v1.2.0"
+	versionRe     = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
+	goarch        string
+	goos          string
+	noupgrade     bool
+	version       string
+	goCmd         string
+	goVersion     float64
+	race          bool
+	debug         = os.Getenv("BUILDDEBUG") != ""
+	extraTags     string
+	installSuffix string
+	pkgdir        string
+	cc            string
+	debugBinary   bool
+	coverage      bool
+	timeout       = "120s"
 )
 
 type target struct {
 	name              string
 	debname           string
 	debdeps           []string
+	debpre            string
 	debpost           string
 	description       string
-	buildPkg          string
+	buildPkgs         []string
 	binaryName        string
 	archiveFiles      []archiveFile
+	systemdServices   []string
 	installationFiles []archiveFile
 	tags              []string
 }
@@ -75,9 +77,8 @@ type archiveFile struct {
 var targets = map[string]target{
 	"all": {
 		// Only valid for the "build" and "install" commands as it lacks all
-		// the archive creation stuff.
-		buildPkg: "github.com/syncthing/syncthing/cmd/...",
-		tags:     []string{"purego"},
+		// the archive creation stuff. buildPkgs gets filled out in init()
+		tags: []string{"purego"},
 	},
 	"syncthing": {
 		// The default target for "build", "install", "tar", "zip", "deb", etc.
@@ -86,7 +87,7 @@ var targets = map[string]target{
 		debdeps:     []string{"libc6", "procps"},
 		debpost:     "script/post-upgrade",
 		description: "Open Source Continuous File Synchronization",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/syncthing",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/syncthing"},
 		binaryName:  "syncthing", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -128,8 +129,9 @@ var targets = map[string]target{
 		name:        "stdiscosrv",
 		debname:     "syncthing-discosrv",
 		debdeps:     []string{"libc6"},
+		debpre:      "cmd/stdiscosrv/scripts/preinst",
 		description: "Syncthing Discovery Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/stdiscosrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/stdiscosrv"},
 		binaryName:  "stdiscosrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -137,12 +139,17 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
+		systemdServices: []string{
+			"cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service",
+		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
 			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0644},
 			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0644},
 			{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-discosrv", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv", dst: "deb/etc/ufw/applications.d/stdiscosrv", perm: 0644},
 		},
 		tags: []string{"purego"},
 	},
@@ -150,8 +157,9 @@ var targets = map[string]target{
 		name:        "strelaysrv",
 		debname:     "syncthing-relaysrv",
 		debdeps:     []string{"libc6"},
+		debpre:      "cmd/strelaysrv/scripts/preinst",
 		description: "Syncthing Relay Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/strelaysrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaysrv"},
 		binaryName:  "strelaysrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -160,6 +168,9 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
+		systemdServices: []string{
+			"cmd/strelaysrv/etc/linux-systemd/strelaysrv.service",
+		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
 			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0644},
@@ -167,6 +178,8 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0644},
 			{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0644},
+			{src: "cmd/strelaysrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-relaysrv", perm: 0644},
+			{src: "cmd/strelaysrv/etc/firewall-ufw/strelaysrv", dst: "deb/etc/ufw/applications.d/strelaysrv", perm: 0644},
 		},
 	},
 	"strelaypoolsrv": {
@@ -174,7 +187,7 @@ var targets = map[string]target{
 		debname:     "syncthing-relaypoolsrv",
 		debdeps:     []string{"libc6"},
 		description: "Syncthing Relay Pool Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/strelaypoolsrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaypoolsrv"},
 		binaryName:  "strelaypoolsrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -200,11 +213,22 @@ type dependencyRepo struct {
 }
 
 var dependencyRepos = []dependencyRepo{
-	{path: "protobuf", repo: "https://github.com/gogo/protobuf.git", commit: gogoProtoVersion},
 	{path: "xdr", repo: "https://github.com/calmh/xdr.git", commit: "08e072f9cb16"},
 }
 
 func init() {
+	all := targets["all"]
+	pkgs, _ := filepath.Glob("cmd/*")
+	for _, pkg := range pkgs {
+		pkg = filepath.Base(pkg)
+		if strings.HasPrefix(pkg, ".") {
+			// ignore dotfiles
+			continue
+		}
+		all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/cmd/%s", pkg))
+	}
+	targets["all"] = all
+
 	// The "syncthing" target includes a few more files found in the "etc"
 	// and "extra" dirs.
 	syncthingPkg := targets["syncthing"]
@@ -370,9 +394,6 @@ func install(target target, tags []string) {
 	}
 	os.Setenv("GOBIN", filepath.Join(cwd, "bin"))
 
-	args := []string{"install", "-v"}
-	args = appendParameters(args, tags, target)
-
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
 	os.Setenv("CC", cc)
@@ -388,19 +409,20 @@ func install(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	runPrint(goCmd, args...)
+	for _, pkg := range target.buildPkgs {
+		args := []string{"install", "-v"}
+		args = appendParameters(args, tags, pkg)
+
+		runPrint(goCmd, args...)
+	}
 }
 
 func build(target target, tags []string) {
 	lazyRebuildAssets()
-
 	tags = append(target.tags, tags...)
 
 	rmr(target.BinaryName())
 
-	args := []string{"build", "-v"}
-	args = appendParameters(args, tags, target)
-
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
 	os.Setenv("CC", cc)
@@ -420,10 +442,15 @@ func build(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	runPrint(goCmd, args...)
+	for _, pkg := range target.buildPkgs {
+		args := []string{"build", "-v"}
+		args = appendParameters(args, tags, pkg)
+
+		runPrint(goCmd, args...)
+	}
 }
 
-func appendParameters(args []string, tags []string, target target) []string {
+func appendParameters(args []string, tags []string, pkg string) []string {
 	if pkgdir != "" {
 		args = append(args, "-pkgdir", pkgdir)
 	}
@@ -439,7 +466,7 @@ func appendParameters(args []string, tags []string, target target) []string {
 
 	if !debugBinary {
 		// Regular binaries get version tagged and skip some debug symbols
-		args = append(args, "-ldflags", ldflags())
+		args = append(args, "-ldflags", ldflags(path.Base(pkg)))
 	} else {
 		// -gcflags to disable optimizations and inlining. Skip -ldflags
 		// because `Could not launch program: decoding dwarf section info at
@@ -448,7 +475,7 @@ func appendParameters(args []string, tags []string, target target) []string {
 		args = append(args, "-gcflags", "-N -l")
 	}
 
-	return append(args, target.buildPkg)
+	return append(args, pkg)
 }
 
 func buildTar(target target) {
@@ -555,9 +582,15 @@ func buildDeb(target target) {
 	for _, dep := range target.debdeps {
 		args = append(args, "-d", dep)
 	}
+	for _, service := range target.systemdServices {
+		args = append(args, "--deb-systemd", service)
+	}
 	if target.debpost != "" {
 		args = append(args, "--after-upgrade", target.debpost)
 	}
+	if target.debpre != "" {
+		args = append(args, "--before-install", target.debpre)
+	}
 	runPrint("fpm", args...)
 }
 
@@ -690,6 +723,7 @@ func listFiles(dir string) []string {
 		if err != nil {
 			return err
 		}
+
 		if fi.Mode().IsRegular() {
 			res = append(res, path)
 		}
@@ -736,14 +770,21 @@ func shouldRebuildAssets(target, srcdir string) bool {
 }
 
 func proto() {
-	runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", gogoProtoVersion))
+	pv := protobufVersion()
+	dependencyRepos = append(dependencyRepos,
+		dependencyRepo{path: "protobuf", repo: "https://github.com/gogo/protobuf.git", commit: pv},
+	)
+
+	runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
 	os.MkdirAll("repos", 0755)
 	for _, dep := range dependencyRepos {
 		path := filepath.Join("repos", dep.path)
 		if _, err := os.Stat(path); err != nil {
 			runPrintInDir("repos", "git", "clone", dep.repo, dep.path)
-			runPrintInDir(path, "git", "checkout", dep.commit)
+		} else {
+			runPrintInDir(path, "git", "fetch")
 		}
+		runPrintInDir(path, "git", "checkout", dep.commit)
 	}
 	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/stdiscosrv")
 }
@@ -764,7 +805,7 @@ func transifex() {
 	runPrint(goCmd, "run", "../../../../script/transifexdl.go")
 }
 
-func ldflags() string {
+func ldflags(program string) string {
 	sep := '='
 	if goVersion > 0 && goVersion < 1.5 {
 		sep = ' '
@@ -776,6 +817,10 @@ func ldflags() string {
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp%c%d", sep, buildStamp())
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User%c%s", sep, buildUser())
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Host%c%s", sep, buildHost())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Program%c%s", sep, program)
+	if v := os.Getenv("EXTRA_LDFLAGS"); v != "" {
+		fmt.Fprintf(b, " %s", v)
+	}
 	return b.String()
 }
 
@@ -1232,3 +1277,11 @@ func (t target) BinaryName() string {
 	}
 	return t.binaryName
 }
+
+func protobufVersion() string {
+	bs, err := runError(goCmd, "list", "-f", "{{.Version}}", "-m", "github.com/gogo/protobuf")
+	if err != nil {
+		log.Fatal("Getting protobuf version:", err)
+	}
+	return string(bs)
+}

cmd/stbench/main.go

@@ -1,143 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// This doesn't build on Windows due to the Rusage stuff.
-
-// +build !windows
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"runtime"
-	"syscall"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/rc"
-)
-
-var homeDir = "h1"
-var syncthingBin = "./bin/syncthing"
-var test = "scan"
-
-func main() {
-	flag.StringVar(&homeDir, "home", homeDir, "Home directory location")
-	flag.StringVar(&syncthingBin, "bin", syncthingBin, "Binary location")
-	flag.StringVar(&test, "test", test, "Test to run")
-	flag.Parse()
-
-	switch test {
-	case "scan":
-		// scan measures the resource usage required to perform the initial
-		// scan, without cleaning away the database first.
-		testScan()
-	}
-}
-
-// testScan starts a process and reports on the resource usage required to
-// perform the initial scan.
-func testScan() {
-	log.Println("Starting...")
-	p := rc.NewProcess("127.0.0.1:8081")
-	if err := p.Start(syncthingBin, "-home", homeDir, "-no-browser"); err != nil {
-		log.Println(err)
-		return
-	}
-	defer p.Stop()
-
-	wallTime := awaitScanComplete(p)
-
-	report(p, wallTime)
-}
-
-// awaitScanComplete waits for a folder to transition idle->scanning and
-// then scanning->idle and returns the time taken for the scan.
-func awaitScanComplete(p *rc.Process) time.Duration {
-	log.Println("Awaiting scan completion...")
-	var t0, t1 time.Time
-	lastEvent := 0
-loop:
-	for {
-		evs, err := p.Events(lastEvent)
-		if err != nil {
-			continue
-		}
-
-		for _, ev := range evs {
-			if ev.Type == "StateChanged" {
-				data := ev.Data.(map[string]interface{})
-				log.Println(ev)
-
-				if data["to"].(string) == "scanning" {
-					t0 = ev.Time
-					continue
-				}
-
-				if !t0.IsZero() && data["to"].(string) == "idle" {
-					t1 = ev.Time
-					break loop
-				}
-			}
-			lastEvent = ev.ID
-		}
-
-		time.Sleep(250 * time.Millisecond)
-	}
-
-	return t1.Sub(t0)
-}
-
-// report stops the given process and reports on its resource usage in two
-// ways: human readable to stderr, and CSV to stdout.
-func report(p *rc.Process, wallTime time.Duration) {
-	sv, err := p.SystemVersion()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	ss, err := p.SystemStatus()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	proc, err := p.Stop()
-	if err != nil {
-		return
-	}
-
-	rusage, ok := proc.SysUsage().(*syscall.Rusage)
-	if !ok {
-		return
-	}
-
-	log.Println("Version:", sv.Version)
-	log.Println("Alloc:", ss.Alloc/1024, "KiB")
-	log.Println("Sys:", ss.Sys/1024, "KiB")
-	log.Println("Goroutines:", ss.Goroutines)
-	log.Println("Wall time:", wallTime)
-	log.Println("Utime:", time.Duration(rusage.Utime.Nano()))
-	log.Println("Stime:", time.Duration(rusage.Stime.Nano()))
-	if runtime.GOOS == "darwin" {
-		// Darwin reports in bytes, Linux seems to report in KiB even
-		// though the manpage says otherwise.
-		rusage.Maxrss /= 1024
-	}
-	log.Println("MaxRSS:", rusage.Maxrss, "KiB")
-
-	fmt.Printf("%s,%d,%d,%d,%.02f,%.02f,%.02f,%d\n",
-		sv.Version,
-		ss.Alloc/1024,
-		ss.Sys/1024,
-		ss.Goroutines,
-		wallTime.Seconds(),
-		time.Duration(rusage.Utime.Nano()).Seconds(),
-		time.Duration(rusage.Stime.Nano()).Seconds(),
-		rusage.Maxrss)
-}

cmd/stcli/main.go

@@ -14,7 +14,6 @@ import (
 	"log"
 	"os"
 	"reflect"
-	"strings"
 
 	"github.com/AudriusButkevicius/recli"
 	"github.com/flynn-archive/go-shlex"
@@ -22,6 +21,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/urfave/cli"
@@ -85,7 +85,7 @@ func main() {
 		myID := protocol.NewDeviceID(cert.Certificate[0])
 
 		// Load the config
-		cfg, err := config.Load(locations.Get(locations.ConfigFile), myID)
+		cfg, err := config.Load(locations.Get(locations.ConfigFile), myID, events.NoopLogger)
 		if err != nil {
 			log.Fatalln(errors.Wrap(err, "loading config"))
 		}
@@ -127,7 +127,7 @@ func main() {
 	app.HelpName = app.Name
 	app.Author = "The Syncthing Authors"
 	app.Usage = "Syncthing command line interface"
-	app.Version = strings.Replace(build.LongVersion, "syncthing", app.Name, 1)
+	app.Version = build.Version
 	app.Flags = fakeFlags
 	app.Metadata = map[string]interface{}{
 		"client": client,

cmd/stcli/utils.go

@@ -12,7 +12,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"text/tabwriter"
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/urfave/cli"
@@ -45,12 +44,6 @@ func dumpOutput(url string) cli.ActionFunc {
 	}
 }
 
-func newTableWriter() *tabwriter.Writer {
-	writer := new(tabwriter.Writer)
-	writer.Init(os.Stdout, 0, 8, 0, '\t', 0)
-	return writer
-}
-
 func getConfig(c *APIClient) (config.Configuration, error) {
 	cfg := config.Configuration{}
 	response, err := c.Get("system/config")

cmd/stcrashreceiver/sentry.go

@@ -12,6 +12,7 @@ import (
 	"io/ioutil"
 	"regexp"
 	"strings"
+	"sync"
 
 	raven "github.com/getsentry/raven-go"
 	"github.com/maruel/panicparse/stack"
@@ -19,15 +20,33 @@ import (
 
 const reportServer = "https://crash.syncthing.net/report/"
 
+var loader = newGithubSourceCodeLoader()
+
+func init() {
+	raven.SetSourceCodeLoader(loader)
+}
+
+var (
+	clients    = make(map[string]*raven.Client)
+	clientsMut sync.Mutex
+)
+
 func sendReport(dsn, path string, report []byte) error {
 	pkt, err := parseReport(path, report)
 	if err != nil {
 		return err
 	}
 
-	cli, err := raven.New(dsn)
-	if err != nil {
-		return err
+	clientsMut.Lock()
+	defer clientsMut.Unlock()
+
+	cli, ok := clients[dsn]
+	if !ok {
+		cli, err = raven.New(dsn)
+		if err != nil {
+			return err
+		}
+		clients[dsn] = cli
 	}
 
 	// The client sets release and such on the packet before sending, in the
@@ -36,6 +55,7 @@ func sendReport(dsn, path string, report []byte) error {
 	cli.SetRelease(pkt.Release)
 	cli.SetEnvironment(pkt.Environment)
 
+	defer cli.Wait()
 	_, errC := cli.Capture(pkt, nil)
 	return <-errC
 }
@@ -80,30 +100,38 @@ func parseReport(path string, report []byte) (*raven.Packet, error) {
 		return nil, err
 	}
 
+	// Lock the source code loader to the version we are processing here.
+	if version.commit != "" {
+		// We have a commit hash, so we know exactly which source to use
+		loader.LockWithVersion(version.commit)
+	} else if strings.HasPrefix(version.tag, "v") {
+		// Lets hope the tag is close enough
+		loader.LockWithVersion(version.tag)
+	} else {
+		// Last resort
+		loader.LockWithVersion("master")
+	}
+	defer loader.Unlock()
+
 	var trace raven.Stacktrace
 	for _, gr := range ctx.Goroutines {
 		if gr.First {
 			trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
 			for i, sc := range gr.Stack.Calls {
-				trace.Frames[len(trace.Frames)-1-i] = &raven.StacktraceFrame{
-					Function: sc.Func.Name(),
-					Module:   sc.Func.PkgName(),
-					Filename: sc.SrcPath,
-					Lineno:   sc.Line,
-				}
+				trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name(), sc.SrcPath, sc.Line, 3, nil)
 			}
 			break
 		}
 	}
 
 	pkt := &raven.Packet{
-		Message:  string(subjectLine),
-		Platform: "go",
-		Release:  version.tag,
+		Message:     string(subjectLine),
+		Platform:    "go",
+		Release:     version.tag,
+		Environment: version.environment(),
 		Tags: raven.Tags{
 			raven.Tag{Key: "version", Value: version.version},
 			raven.Tag{Key: "tag", Value: version.tag},
-			raven.Tag{Key: "commit", Value: version.commit},
 			raven.Tag{Key: "codename", Value: version.codename},
 			raven.Tag{Key: "runtime", Value: version.runtime},
 			raven.Tag{Key: "goos", Value: version.goos},
@@ -115,6 +143,9 @@ func parseReport(path string, report []byte) (*raven.Packet, error) {
 		},
 		Interfaces: []raven.Interface{&trace},
 	}
+	if version.commit != "" {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
+	}
 
 	return pkt, nil
 }
@@ -133,6 +164,19 @@ type version struct {
 	builder  string // "jb@kvin.kastelo.net"
 }
 
+func (v version) environment() string {
+	if v.commit != "" {
+		return "Development"
+	}
+	if strings.Contains(v.tag, "-rc.") {
+		return "Candidate"
+	}
+	if strings.Contains(v.tag, "-") {
+		return "Beta"
+	}
+	return "Stable"
+}
+
 func parseVersion(line string) (version, error) {
 	m := longVersionRE.FindStringSubmatch(line)
 	if len(m) == 0 {

cmd/stcrashreceiver/sourcecodeloader.go

@@ -0,0 +1,114 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	urlPrefix   = "https://raw.githubusercontent.com/syncthing/syncthing/"
+	httpTimeout = 10 * time.Second
+)
+
+type githubSourceCodeLoader struct {
+	mut     sync.Mutex
+	version string
+	cache   map[string]map[string][][]byte // version -> file -> lines
+	client  *http.Client
+}
+
+func newGithubSourceCodeLoader() *githubSourceCodeLoader {
+	return &githubSourceCodeLoader{
+		cache:  make(map[string]map[string][][]byte),
+		client: &http.Client{Timeout: httpTimeout},
+	}
+}
+
+func (l *githubSourceCodeLoader) LockWithVersion(version string) {
+	l.mut.Lock()
+	l.version = version
+	if _, ok := l.cache[version]; !ok {
+		l.cache[version] = make(map[string][][]byte)
+	}
+}
+
+func (l *githubSourceCodeLoader) Unlock() {
+	l.mut.Unlock()
+}
+
+func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]byte, int) {
+	filename = filepath.ToSlash(filename)
+	lines, ok := l.cache[l.version][filename]
+	if !ok {
+		// Cache whatever we managed to find (or nil if nothing, so we don't try again)
+		defer func() {
+			l.cache[l.version][filename] = lines
+		}()
+
+		knownPrefixes := []string{"/lib/", "/cmd/"}
+		var idx int
+		for _, pref := range knownPrefixes {
+			idx = strings.Index(filename, pref)
+			if idx >= 0 {
+				break
+			}
+		}
+		if idx == -1 {
+			return nil, 0
+		}
+
+		url := urlPrefix + l.version + filename[idx:]
+		resp, err := l.client.Get(url)
+
+		if err != nil || resp.StatusCode != http.StatusOK {
+			fmt.Println("Loading source:", err.Error())
+			return nil, 0
+		}
+		data, err := ioutil.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		if err != nil {
+			fmt.Println("Loading source:", err.Error())
+			return nil, 0
+		}
+		lines = bytes.Split(data, []byte{'\n'})
+	}
+
+	return getLineFromLines(lines, line, context)
+}
+
+func getLineFromLines(lines [][]byte, line, context int) ([][]byte, int) {
+	if lines == nil {
+		// cached error from ReadFile: return no lines
+		return nil, 0
+	}
+
+	line-- // stack trace lines are 1-indexed
+	start := line - context
+	var idx int
+	if start < 0 {
+		start = 0
+		idx = line
+	} else {
+		idx = context
+	}
+	end := line + context + 1
+	if line >= len(lines) {
+		return nil, 0
+	}
+	if end > len(lines) {
+		end = len(lines)
+	}
+	return lines[start:end], idx
+}

cmd/stcrashreceiver/stcrashreceiver.go

@@ -13,6 +13,8 @@
 package main
 
 import (
+	"bytes"
+	"compress/gzip"
 	"flag"
 	"io"
 	"io/ioutil"
@@ -52,12 +54,12 @@ func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	// The final path component should be a SHA256 hash in hex, so 64 hex
 	// characters. We don't care about case on the request but use lower
 	// case internally.
-	base := strings.ToLower(path.Base(req.URL.Path))
-	if len(base) != 64 {
+	reportID := strings.ToLower(path.Base(req.URL.Path))
+	if len(reportID) != 64 {
 		http.Error(w, "Bad request", http.StatusBadRequest)
 		return
 	}
-	for _, c := range base {
+	for _, c := range reportID {
 		if c >= 'a' && c <= 'f' {
 			continue
 		}
@@ -68,40 +70,57 @@ func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
+	// The location of the report on disk, compressed
+	fullPath := filepath.Join(r.dir, r.dirFor(reportID), reportID) + ".gz"
+
 	switch req.Method {
+	case http.MethodGet:
+		r.serveGet(fullPath, w, req)
 	case http.MethodHead:
-		r.serveHead(base, w, req)
+		r.serveHead(fullPath, w, req)
 	case http.MethodPut:
-		r.servePut(base, w, req)
+		r.servePut(reportID, fullPath, w, req)
 	default:
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 	}
 }
 
+// serveGet responds to GET requests by serving the uncompressed report.
+func (r *crashReceiver) serveGet(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	fd, err := os.Open(fullPath)
+	if err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+		return
+	}
+
+	defer fd.Close()
+	gr, err := gzip.NewReader(fd)
+	if err != nil {
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+	_, _ = io.Copy(w, gr) // best effort
+}
+
 // serveHead responds to HEAD requests by checking if the named report
 // already exists in the system.
-func (r *crashReceiver) serveHead(base string, w http.ResponseWriter, _ *http.Request) {
-	path := filepath.Join(r.dirFor(base), base)
-	if _, err := os.Lstat(path); err != nil {
+func (r *crashReceiver) serveHead(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	if _, err := os.Lstat(fullPath); err != nil {
 		http.Error(w, "Not found", http.StatusNotFound)
 	}
-	// 200 OK
 }
 
 // servePut accepts and stores the given report.
-func (r *crashReceiver) servePut(base string, w http.ResponseWriter, req *http.Request) {
-	path := filepath.Join(r.dirFor(base), base)
-	fullPath := filepath.Join(r.dir, path)
-
+func (r *crashReceiver) servePut(reportID, fullPath string, w http.ResponseWriter, req *http.Request) {
 	// Ensure the destination directory exists
 	if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
-		log.Printf("Creating directory for report %s: %v", base, err)
+		log.Println("Creating directory:", err)
 		http.Error(w, "Internal server error", http.StatusInternalServerError)
 		return
 	}
 
 	// Read at most maxRequestSize of report data.
-	log.Println("Receiving report", base)
+	log.Println("Receiving report", reportID)
 	lr := io.LimitReader(req.Body, maxRequestSize)
 	bs, err := ioutil.ReadAll(lr)
 	if err != nil {
@@ -110,10 +129,16 @@ func (r *crashReceiver) servePut(base string, w http.ResponseWriter, req *http.R
 		return
 	}
 
-	// Create an output file
-	err = ioutil.WriteFile(fullPath, bs, 0644)
+	// Compress the report for storage
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	_, _ = gw.Write(bs) // can't fail
+	gw.Close()
+
+	// Create an output file with the compressed report
+	err = ioutil.WriteFile(fullPath, buf.Bytes(), 0644)
 	if err != nil {
-		log.Printf("Creating file for report %s: %v", base, err)
+		log.Println("Saving report:", err)
 		http.Error(w, "Internal server error", http.StatusInternalServerError)
 		return
 	}
@@ -122,7 +147,7 @@ func (r *crashReceiver) servePut(base string, w http.ResponseWriter, req *http.R
 	if r.dsn != "" {
 		go func() {
 			// There's no need for the client to have to wait for this part.
-			if err := sendReport(r.dsn, path, bs); err != nil {
+			if err := sendReport(r.dsn, reportID, bs); err != nil {
 				log.Println("Failed to send report:", err)
 			}
 		}()

cmd/stdiscosrv/apisrv.go

@@ -18,6 +18,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -279,6 +280,10 @@ func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addre
 		dbAddrs[i].Expires = expire
 	}
 
+	// The address slice must always be sorted for database merges to work
+	// properly.
+	sort.Sort(databaseAddressOrder(dbAddrs))
+
 	seen := now.UnixNano()
 	if s.repl != nil {
 		s.repl.send(key, dbAddrs, seen)
@@ -295,28 +300,66 @@ func certificateBytes(req *http.Request) []byte {
 		return req.TLS.PeerCertificates[0].Raw
 	}
 
+	var bs []byte
+
 	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
-		bs := []byte(hdr)
-		// The certificate is in PEM format but with spaces for newlines. We
-		// need to reinstate the newlines for the PEM decoder. But we need to
-		// leave the spaces in the BEGIN and END lines - the first and last
-		// space - alone.
-		firstSpace := bytes.Index(bs, []byte(" "))
-		lastSpace := bytes.LastIndex(bs, []byte(" "))
-		for i := firstSpace + 1; i < lastSpace; i++ {
-			if bs[i] == ' ' {
-				bs[i] = '\n'
+		if strings.Contains(hdr, "%") {
+			// Nginx using $ssl_client_escaped_cert
+			// The certificate is in PEM format with url encoding.
+			// We need to decode for the PEM decoder
+			hdr, err := url.QueryUnescape(hdr)
+			if err != nil {
+				// Decoding failed
+				return nil
+			}
+
+			bs = []byte(hdr)
+		} else {
+			// Nginx using $ssl_client_cert
+			// The certificate is in PEM format but with spaces for newlines. We
+			// need to reinstate the newlines for the PEM decoder. But we need to
+			// leave the spaces in the BEGIN and END lines - the first and last
+			// space - alone.
+			bs = []byte(hdr)
+			firstSpace := bytes.Index(bs, []byte(" "))
+			lastSpace := bytes.LastIndex(bs, []byte(" "))
+			for i := firstSpace + 1; i < lastSpace; i++ {
+				if bs[i] == ' ' {
+					bs[i] = '\n'
+				}
 			}
 		}
-		block, _ := pem.Decode(bs)
-		if block == nil {
+	} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
+		// Traefik 2 passtlsclientcert
+		// The certificate is in PEM format with url encoding but without newlines
+		// and start/end statements. We need to decode, reinstate the newlines every 64
+		// character and add statements for the PEM decoder
+		hdr, err := url.QueryUnescape(hdr)
+		if err != nil {
 			// Decoding failed
 			return nil
 		}
-		return block.Bytes
+
+		for i := 64; i < len(hdr); i += 65 {
+			hdr = hdr[:i] + "\n" + hdr[i:]
+		}
+
+		hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
+		hdr = hdr + "\n-----END CERTIFICATE-----\n"
+		bs = []byte(hdr)
 	}
 
-	return nil
+	if bs == nil {
+		return nil
+	}
+
+	block, _ := pem.Decode(bs)
+	if block == nil {
+		// Decoding failed
+		return nil
+	}
+
+	return block.Bytes
 }
 
 // fixupAddresses checks the list of addresses, removing invalid ones and

cmd/stdiscosrv/database.go

@@ -10,6 +10,7 @@
 package main
 
 import (
+	"log"
 	"sort"
 	"time"
 
@@ -263,12 +264,15 @@ func (s *levelDBStore) Stop() {
 // chosen for any duplicates.
 func merge(a, b DatabaseRecord) DatabaseRecord {
 	// Both lists must be sorted for this to work.
-	sort.Slice(a.Addresses, func(i, j int) bool {
-		return a.Addresses[i].Address < a.Addresses[j].Address
-	})
-	sort.Slice(b.Addresses, func(i, j int) bool {
-		return b.Addresses[i].Address < b.Addresses[j].Address
-	})
+	if !sort.IsSorted(databaseAddressOrder(a.Addresses)) {
+		log.Println("Warning: bug: addresses not correctly sorted in merge")
+		a.Addresses = sortedAddressCopy(a.Addresses)
+	}
+	if !sort.IsSorted(databaseAddressOrder(b.Addresses)) {
+		// no warning because this is the side we read from disk and it may
+		// legitimately predate correct sorting.
+		b.Addresses = sortedAddressCopy(b.Addresses)
+	}
 
 	res := DatabaseRecord{
 		Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
@@ -352,3 +356,24 @@ func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
 	}
 	return addrs
 }
+
+func sortedAddressCopy(addrs []DatabaseAddress) []DatabaseAddress {
+	sorted := make([]DatabaseAddress, len(addrs))
+	copy(sorted, addrs)
+	sort.Sort(databaseAddressOrder(sorted))
+	return sorted
+}
+
+type databaseAddressOrder []DatabaseAddress
+
+func (s databaseAddressOrder) Less(a, b int) bool {
+	return s[a].Address < s[b].Address
+}
+
+func (s databaseAddressOrder) Swap(a, b int) {
+	s[a], s[b] = s[b], s[a]
+}
+
+func (s databaseAddressOrder) Len() int {
+	return len(s)
+}

cmd/stdiscosrv/database.pb.go

@@ -3,12 +3,14 @@
 
 package main
 
-import proto "github.com/gogo/protobuf/proto"
-import fmt "fmt"
-import math "math"
-import _ "github.com/gogo/protobuf/gogoproto"
-
-import io "io"
+import (
+	fmt "fmt"
+	_ "github.com/gogo/protobuf/gogoproto"
+	proto "github.com/gogo/protobuf/proto"
+	io "io"
+	math "math"
+	math_bits "math/bits"
+)
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
@@ -19,7 +21,7 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
 
 type DatabaseRecord struct {
 	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
@@ -32,7 +34,7 @@ func (m *DatabaseRecord) Reset()         { *m = DatabaseRecord{} }
 func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
 func (*DatabaseRecord) ProtoMessage()    {}
 func (*DatabaseRecord) Descriptor() ([]byte, []int) {
-	return fileDescriptor_database_0f49e029703a04f5, []int{0}
+	return fileDescriptor_b90fe3356ea5df07, []int{0}
 }
 func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -42,15 +44,15 @@ func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, erro
 		return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalTo(b)
+		n, err := m.MarshalToSizedBuffer(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (dst *DatabaseRecord) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DatabaseRecord.Merge(dst, src)
+func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseRecord.Merge(m, src)
 }
 func (m *DatabaseRecord) XXX_Size() int {
 	return m.Size()
@@ -71,7 +73,7 @@ func (m *ReplicationRecord) Reset()         { *m = ReplicationRecord{} }
 func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
 func (*ReplicationRecord) ProtoMessage()    {}
 func (*ReplicationRecord) Descriptor() ([]byte, []int) {
-	return fileDescriptor_database_0f49e029703a04f5, []int{1}
+	return fileDescriptor_b90fe3356ea5df07, []int{1}
 }
 func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -81,15 +83,15 @@ func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, e
 		return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalTo(b)
+		n, err := m.MarshalToSizedBuffer(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (dst *ReplicationRecord) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ReplicationRecord.Merge(dst, src)
+func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ReplicationRecord.Merge(m, src)
 }
 func (m *ReplicationRecord) XXX_Size() int {
 	return m.Size()
@@ -109,7 +111,7 @@ func (m *DatabaseAddress) Reset()         { *m = DatabaseAddress{} }
 func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
 func (*DatabaseAddress) ProtoMessage()    {}
 func (*DatabaseAddress) Descriptor() ([]byte, []int) {
-	return fileDescriptor_database_0f49e029703a04f5, []int{2}
+	return fileDescriptor_b90fe3356ea5df07, []int{2}
 }
 func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -119,15 +121,15 @@ func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, err
 		return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalTo(b)
+		n, err := m.MarshalToSizedBuffer(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (dst *DatabaseAddress) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DatabaseAddress.Merge(dst, src)
+func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseAddress.Merge(m, src)
 }
 func (m *DatabaseAddress) XXX_Size() int {
 	return m.Size()
@@ -143,10 +145,34 @@ func init() {
 	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
 	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
 }
+
+func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
+
+var fileDescriptor_b90fe3356ea5df07 = []byte{
+	// 270 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
+	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
+	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
+	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
+	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
+	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
+	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
+	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
+	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
+	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
+	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
+	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
+	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
+	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
+	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
+	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
+	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
+}
+
 func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -154,44 +180,51 @@ func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Addresses) > 0 {
-		for _, msg := range m.Addresses {
-			dAtA[i] = 0xa
-			i++
-			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
-			n, err := msg.MarshalTo(dAtA[i:])
-			if err != nil {
-				return 0, err
-			}
-			i += n
-		}
-	}
-	if m.Misses != 0 {
-		dAtA[i] = 0x10
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+	if m.Missed != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+		i--
+		dAtA[i] = 0x20
 	}
 	if m.Seen != 0 {
-		dAtA[i] = 0x18
-		i++
 		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
 	}
-	if m.Missed != 0 {
-		dAtA[i] = 0x20
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+	if m.Misses != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+		i--
+		dAtA[i] = 0x10
 	}
-	return i, nil
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
 }
 
 func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -199,40 +232,48 @@ func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Key) > 0 {
-		dAtA[i] = 0xa
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
-		i += copy(dAtA[i:], m.Key)
+	if m.Seen != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
 	}
 	if len(m.Addresses) > 0 {
-		for _, msg := range m.Addresses {
-			dAtA[i] = 0x12
-			i++
-			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
-			n, err := msg.MarshalTo(dAtA[i:])
-			if err != nil {
-				return 0, err
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
 			}
-			i += n
+			i--
+			dAtA[i] = 0x12
 		}
 	}
-	if m.Seen != 0 {
-		dAtA[i] = 0x18
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+	if len(m.Key) > 0 {
+		i -= len(m.Key)
+		copy(dAtA[i:], m.Key)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		i--
+		dAtA[i] = 0xa
 	}
-	return i, nil
+	return len(dAtA) - i, nil
 }
 
 func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -240,32 +281,40 @@ func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Address) > 0 {
-		dAtA[i] = 0xa
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
-		i += copy(dAtA[i:], m.Address)
-	}
 	if m.Expires != 0 {
-		dAtA[i] = 0x10
-		i++
 		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
+		i--
+		dAtA[i] = 0x10
 	}
-	return i, nil
+	if len(m.Address) > 0 {
+		i -= len(m.Address)
+		copy(dAtA[i:], m.Address)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
 }
 
 func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
+	offset -= sovDatabase(v)
+	base := offset
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
 		v >>= 7
 		offset++
 	}
 	dAtA[offset] = uint8(v)
-	return offset + 1
+	return base
 }
 func (m *DatabaseRecord) Size() (n int) {
 	if m == nil {
@@ -330,14 +379,7 @@ func (m *DatabaseAddress) Size() (n int) {
 }
 
 func sovDatabase(x uint64) (n int) {
-	for {
-		n++
-		x >>= 7
-		if x == 0 {
-			break
-		}
-	}
-	return n
+	return (math_bits.Len64(x|1) + 6) / 7
 }
 func sozDatabase(x uint64) (n int) {
 	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
@@ -357,7 +399,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -385,7 +427,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= (int(b) & 0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -394,6 +436,9 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -416,7 +461,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Misses |= (int32(b) & 0x7F) << shift
+				m.Misses |= int32(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -435,7 +480,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= (int64(b) & 0x7F) << shift
+				m.Seen |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -454,7 +499,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Missed |= (int64(b) & 0x7F) << shift
+				m.Missed |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -468,6 +513,9 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -495,7 +543,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -523,7 +571,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -533,6 +581,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -552,7 +603,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= (int(b) & 0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -561,6 +612,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -583,7 +637,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= (int64(b) & 0x7F) << shift
+				m.Seen |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -597,6 +651,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -624,7 +681,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -652,7 +709,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -662,6 +719,9 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -681,7 +741,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Expires |= (int64(b) & 0x7F) << shift
+				m.Expires |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -695,6 +755,9 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -710,6 +773,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 func skipDatabase(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0
+	depth := 0
 	for iNdEx < l {
 		var wire uint64
 		for shift := uint(0); ; shift += 7 {
@@ -741,10 +805,8 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			return iNdEx, nil
 		case 1:
 			iNdEx += 8
-			return iNdEx, nil
 		case 2:
 			var length int
 			for shift := uint(0); ; shift += 7 {
@@ -761,76 +823,34 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			iNdEx += length
 			if length < 0 {
 				return 0, ErrInvalidLengthDatabase
 			}
-			return iNdEx, nil
+			iNdEx += length
 		case 3:
-			for {
-				var innerWire uint64
-				var start int = iNdEx
-				for shift := uint(0); ; shift += 7 {
-					if shift >= 64 {
-						return 0, ErrIntOverflowDatabase
-					}
-					if iNdEx >= l {
-						return 0, io.ErrUnexpectedEOF
-					}
-					b := dAtA[iNdEx]
-					iNdEx++
-					innerWire |= (uint64(b) & 0x7F) << shift
-					if b < 0x80 {
-						break
-					}
-				}
-				innerWireType := int(innerWire & 0x7)
-				if innerWireType == 4 {
-					break
-				}
-				next, err := skipDatabase(dAtA[start:])
-				if err != nil {
-					return 0, err
-				}
-				iNdEx = start + next
-			}
-			return iNdEx, nil
+			depth++
 		case 4:
-			return iNdEx, nil
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupDatabase
+			}
+			depth--
 		case 5:
 			iNdEx += 4
-			return iNdEx, nil
 		default:
 			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
 		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthDatabase
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
 	}
-	panic("unreachable")
+	return 0, io.ErrUnexpectedEOF
 }
 
 var (
-	ErrInvalidLengthDatabase = fmt.Errorf("proto: negative length found during unmarshaling")
-	ErrIntOverflowDatabase   = fmt.Errorf("proto: integer overflow")
+	ErrInvalidLengthDatabase        = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowDatabase          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupDatabase = fmt.Errorf("proto: unexpected end of group")
 )
-
-func init() { proto.RegisterFile("database.proto", fileDescriptor_database_0f49e029703a04f5) }
-
-var fileDescriptor_database_0f49e029703a04f5 = []byte{
-	// 270 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
-	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
-	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
-	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
-	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
-	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
-	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
-	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
-	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
-	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
-	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
-	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
-	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
-	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
-	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
-	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
-	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
-}

cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv

@@ -0,0 +1,4 @@
+[stdiscosrv]
+title=Syncthing discovery server
+description=Lets syncthing clients discover each other
+ports=8443/tcp

cmd/stdiscosrv/etc/linux-systemd/default

@@ -0,0 +1,3 @@
+# Default settings for syncthing-relaysrv (strelaysrv).
+## Add Options here:
+DISCOSRV_OPTS=

cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=Syncthing Discovery Server
+After=network.target
+Documentation=man:stdiscosrv(1)
+
+[Service]
+WorkingDirectory=/var/lib/syncthing-discosrv
+EnvironmentFile=/etc/default/syncthing-discosrv
+ExecStart=/usr/bin/stdiscosrv $DISCOSRV_OPTS
+
+# Hardening
+User=syncthing-discosrv
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-discosrv
+NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
+
+[Install]
+WantedBy=multi-user.target
+Alias=syncthing-discosrv.service

cmd/stdiscosrv/main.go

@@ -9,17 +9,15 @@ package main
 import (
 	"crypto/tls"
 	"flag"
-	"fmt"
 	"log"
 	"net"
 	"net/http"
 	"os"
-	"runtime"
-	"strconv"
 	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -65,24 +63,6 @@ var levelDBOptions = &opt.Options{
 	WriteBuffer: 32 << 20, // default 4<<20
 }
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`stdiscosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
-}
-
 var (
 	debug = false
 )
@@ -109,14 +89,18 @@ func main() {
 	flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
 	flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
 	flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
-	log.Println(LongVersion)
+	log.Println(build.LongVersion)
+	if *showVersion {
+		return
+	}
 
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}

cmd/stdiscosrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-discosrv --ingroup syncthing syncthing-discosrv

cmd/stfinddevice/main.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -82,7 +83,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 }
 
 func checkServer(deviceID protocol.DeviceID, server string) checkResult {
-	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger)
 	if err != nil {
 		return checkResult{error: err}
 	}

cmd/strelaypoolsrv/main.go

@@ -633,7 +633,7 @@ func createTestCertificate() tls.Certificate {
 	}
 
 	certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
-	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv")
+	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 20*365)
 	if err != nil {
 		log.Fatalln("Failed to create test X509 key pair:", err)
 	}

cmd/strelaysrv/etc/firewall-ufw/strelaysrv

@@ -0,0 +1,9 @@
+[strelaysrv]
+title=Syncthing relay server
+description=Proxies traffic of syncthing client behind firewalls
+ports=22067/tcp
+
+[strelaysrv-metrics]
+title=Syncthing relay metrics
+description=Provides metrics about the syncthing relay server
+ports=22070/tcp

cmd/strelaysrv/etc/linux-systemd/default

@@ -0,0 +1,5 @@
+# Default settings for syncthing-relaysrv (strelaysrv).
+NAT=true
+
+## Add Options here:
+RELAYSRV_OPTS=

cmd/strelaysrv/etc/linux-systemd/strelaysrv.service

@@ -1,17 +1,25 @@
 [Unit]
-Description=Syncthing relay server
+Description=Syncthing Relay Server
 After=network.target
+Documentation=man:strelaysrv(1)
 
 [Service]
-User=strelaysrv
-Group=strelaysrv
-ExecStart=/usr/bin/strelaysrv
-WorkingDirectory=/var/lib/strelaysrv
+WorkingDirectory=/var/lib/syncthing-relaysrv
+EnvironmentFile=/etc/default/syncthing-relaysrv
+ExecStart=/usr/bin/strelaysrv -nat=${NAT} $RELAYSRV_OPTS
 
-PrivateTmp=true
-ProtectSystem=full
-ProtectHome=true
+# Hardening
+User=syncthing-relaysrv
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-relaysrv
 NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
 
 [Install]
 WantedBy=multi-user.target
+Alias=syncthing-relaysrv.service

cmd/strelaysrv/main.go

@@ -14,12 +14,13 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
 	"sync/atomic"
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/relay/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -33,24 +34,6 @@ import (
 	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
 )
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`strelaysrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
-}
-
 var (
 	listen string
 	debug  bool
@@ -116,8 +99,14 @@ func main() {
 	flag.IntVar(&natTimeout, "nat-timeout", 10, "NAT discovery timeout in seconds")
 	flag.BoolVar(&pprofEnabled, "pprof", false, "Enable the built in profiling on the status server")
 	flag.IntVar(&networkBufferSize, "network-buffer", 2048, "Network buffer size (two of these per proxied connection)")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
+	if *showVersion {
+		fmt.Println(build.LongVersion)
+		return
+	}
+
 	if extAddress == "" {
 		extAddress = listen
 	}
@@ -146,7 +135,7 @@ func main() {
 		}
 	}
 
-	log.Println(LongVersion)
+	log.Println(build.LongVersion)
 
 	maxDescriptors, err := osutil.MaximizeOpenFileLimit()
 	if maxDescriptors > 0 {
@@ -166,7 +155,7 @@ func main() {
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
@@ -194,7 +183,7 @@ func main() {
 		log.Println("ID:", id)
 	}
 
-	wrapper := config.Wrap("config", config.New(id))
+	wrapper := config.Wrap("config", config.New(id), events.NoopLogger)
 	wrapper.SetOptions(config.OptionsConfiguration{
 		NATLeaseM:   natLease,
 		NATRenewalM: natRenewal,

cmd/strelaysrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-relaysrv --ingroup syncthing syncthing-relaysrv

cmd/strelaysrv/status.go

@@ -10,6 +10,8 @@ import (
 	"runtime"
 	"sync/atomic"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/build"
 )
 
 var rc *rateCalculator
@@ -40,10 +42,10 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 
 	sessionMut.Lock()
 	// This can potentially be double the number of pending sessions, as each session has two keys, one for each side.
-	status["version"] = Version
-	status["buildHost"] = BuildHost
-	status["buildUser"] = BuildUser
-	status["buildDate"] = BuildDate
+	status["version"] = build.Version
+	status["buildHost"] = build.Host
+	status["buildUser"] = build.User
+	status["buildDate"] = build.Date
 	status["startTime"] = rc.startTime
 	status["uptimeSeconds"] = time.Since(rc.startTime) / time.Second
 	status["numPendingSessionKeys"] = len(pendingSessions)
@@ -86,9 +88,9 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 }
 
 type rateCalculator struct {
+	counter   *int64 // atomic, must remain 64-bit aligned
 	rates     []int64
 	prev      int64
-	counter   *int64
 	startTime time.Time
 }
 

cmd/stupgrades/main.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"os"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/upgrade"
+)
+
+const defaultURL = "https://api.github.com/repos/syncthing/syncthing/releases?per_page=25"
+
+func main() {
+	url := flag.String("u", defaultURL, "GitHub releases url")
+	flag.Parse()
+
+	rels := upgrade.FetchLatestReleases(*url, "")
+	if rels == nil {
+		// An error was already logged
+		os.Exit(1)
+	}
+
+	sort.Sort(upgrade.SortByRelease(rels))
+	rels = filterForLatest(rels)
+
+	if err := json.NewEncoder(os.Stdout).Encode(rels); err != nil {
+		os.Exit(1)
+	}
+}
+
+// filterForLatest returns the latest stable and prerelease only. If the
+// stable version is newer (comes first in the list) there is no need to go
+// looking for a prerelease at all.
+func filterForLatest(rels []upgrade.Release) []upgrade.Release {
+	var filtered []upgrade.Release
+	var havePre bool
+	for _, rel := range rels {
+		if !rel.Prerelease {
+			// We found a stable version, we're good now.
+			filtered = append(filtered, rel)
+			break
+		}
+		if rel.Prerelease && !havePre {
+			// We remember the first prerelease we find.
+			filtered = append(filtered, rel)
+			havePre = true
+		}
+	}
+	return filtered
+}

cmd/syncthing/auditservice.go

@@ -1,69 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"encoding/json"
-	"io"
-
-	"github.com/syncthing/syncthing/lib/events"
-)
-
-// The auditService subscribes to events and writes these in JSON format, one
-// event per line, to the specified writer.
-type auditService struct {
-	w       io.Writer     // audit destination
-	stop    chan struct{} // signals time to stop
-	started chan struct{} // signals startup complete
-	stopped chan struct{} // signals stop complete
-}
-
-func newAuditService(w io.Writer) *auditService {
-	return &auditService{
-		w:       w,
-		stop:    make(chan struct{}),
-		started: make(chan struct{}),
-		stopped: make(chan struct{}),
-	}
-}
-
-// Serve runs the audit service.
-func (s *auditService) Serve() {
-	defer close(s.stopped)
-	sub := events.Default.Subscribe(events.AllEvents)
-	defer events.Default.Unsubscribe(sub)
-	enc := json.NewEncoder(s.w)
-
-	// We're ready to start processing events.
-	close(s.started)
-
-	for {
-		select {
-		case ev := <-sub.C():
-			enc.Encode(ev)
-		case <-s.stop:
-			return
-		}
-	}
-}
-
-// Stop stops the audit service.
-func (s *auditService) Stop() {
-	close(s.stop)
-}
-
-// WaitForStart returns once the audit service is ready to receive events, or
-// immediately if it's already running.
-func (s *auditService) WaitForStart() {
-	<-s.started
-}
-
-// WaitForStop returns once the audit service has stopped.
-// (Needed by the tests.)
-func (s *auditService) WaitForStop() {
-	<-s.stopped
-}

cmd/syncthing/blockprof.go

@@ -22,11 +22,15 @@ func init() {
 			panic("Couldn't find block profiler")
 		}
 		l.Debugln("Starting block profiling")
-		go saveBlockingProfiles(profiler)
+		go func() {
+			err := saveBlockingProfiles(profiler) // Only returns on error
+			l.Warnln("Block profiler failed:", err)
+			panic("Block profiler failed")
+		}()
 	}
 }
 
-func saveBlockingProfiles(profiler *pprof.Profile) {
+func saveBlockingProfiles(profiler *pprof.Profile) error {
 	runtime.SetBlockProfileRate(1)
 
 	t0 := time.Now()
@@ -35,16 +39,16 @@ func saveBlockingProfiles(profiler *pprof.Profile) {
 
 		fd, err := os.Create(fmt.Sprintf("block-%05d-%07d.pprof", syscall.Getpid(), startms))
 		if err != nil {
-			panic(err)
+			return err
 		}
 		err = profiler.WriteTo(fd, 0)
 		if err != nil {
-			panic(err)
+			return err
 		}
 		err = fd.Close()
 		if err != nil {
-			panic(err)
+			return err
 		}
-
 	}
+	return nil
 }

cmd/syncthing/debug.go

@@ -7,16 +7,9 @@
 package main
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
 	l = logger.DefaultLogger.NewFacility("main", "Main package")
 )
-
-func init() {
-	l.SetDebug("main", strings.Contains(os.Getenv("STTRACE"), "main") || os.Getenv("STTRACE") == "all")
-}

cmd/syncthing/heapprof.go

@@ -23,11 +23,15 @@ func init() {
 			rate = i
 		}
 		l.Debugln("Starting heap profiling")
-		go saveHeapProfiles(rate)
+		go func() {
+			err := saveHeapProfiles(rate) // Only returns on error
+			l.Warnln("Heap profiler failed:", err)
+			panic("Heap profiler failed")
+		}()
 	}
 }
 
-func saveHeapProfiles(rate int) {
+func saveHeapProfiles(rate int) error {
 	runtime.MemProfileRate = rate
 	var memstats, prevMemstats runtime.MemStats
 
@@ -38,21 +42,21 @@ func saveHeapProfiles(rate int) {
 		if memstats.HeapInuse > prevMemstats.HeapInuse {
 			fd, err := os.Create(name + ".tmp")
 			if err != nil {
-				panic(err)
+				return err
 			}
 			err = pprof.WriteHeapProfile(fd)
 			if err != nil {
-				panic(err)
+				return err
 			}
 			err = fd.Close()
 			if err != nil {
-				panic(err)
+				return err
 			}
 
 			os.Remove(name) // Error deliberately ignored
 			err = os.Rename(name+".tmp", name)
 			if err != nil {
-				panic(err)
+				return err
 			}
 
 			prevMemstats = memstats

cmd/syncthing/main.go

@@ -25,52 +25,30 @@ import (
 	"runtime/pprof"
 	"sort"
 	"strconv"
-	"strings"
 	"syscall"
 	"time"
 
-	"github.com/syncthing/syncthing/lib/api"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/connections"
-	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/dialer"
-	"github.com/syncthing/syncthing/lib/discover"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/logger"
-	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/rand"
-	"github.com/syncthing/syncthing/lib/sha256"
+	"github.com/syncthing/syncthing/lib/syncthing"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/upgrade"
-	"github.com/syncthing/syncthing/lib/ur"
 
 	"github.com/pkg/errors"
-	"github.com/thejerf/suture"
 )
 
 const (
-	exitSuccess            = 0
-	exitError              = 1
-	exitNoUpgradeAvailable = 2
-	exitRestarting         = 3
-	exitUpgrading          = 4
+	tlsDefaultCommonName   = "syncthing"
+	deviceCertLifetimeDays = 20 * 365
 )
 
-const (
-	bepProtocolName      = "bep/1.0"
-	tlsDefaultCommonName = "syncthing"
-	maxSystemErrors      = 5
-	initialSystemLog     = 10
-	maxSystemLog         = 250
-)
-
-var myID protocol.DeviceID
-
 const (
 	usage      = "syncthing [options]"
 	extraUsage = `
@@ -158,15 +136,14 @@ The following are valid values for the STTRACE variable:
 
 // Environment options
 var (
-	noUpgradeFromEnv = os.Getenv("STNOUPGRADE") != ""
-	innerProcess     = os.Getenv("STNORESTART") != "" || os.Getenv("STMONITORED") != ""
-	noDefaultFolder  = os.Getenv("STNODEFAULTFOLDER") != ""
+	innerProcess    = os.Getenv("STNORESTART") != "" || os.Getenv("STMONITORED") != ""
+	noDefaultFolder = os.Getenv("STNODEFAULTFOLDER") != ""
 )
 
 type RuntimeOptions struct {
+	syncthing.Options
 	confDir          string
 	resetDatabase    bool
-	resetDeltaIdxs   bool
 	showVersion      bool
 	showPaths        bool
 	showDeviceId     bool
@@ -179,15 +156,12 @@ type RuntimeOptions struct {
 	logFile          string
 	auditEnabled     bool
 	auditFile        string
-	verbose          bool
 	paused           bool
 	unpaused         bool
 	guiAddress       string
 	guiAPIKey        string
 	generateDir      string
 	noRestart        bool
-	profiler         string
-	assetDir         string
 	cpuProfile       bool
 	stRestarting     bool
 	logFlags         int
@@ -197,9 +171,12 @@ type RuntimeOptions struct {
 
 func defaultRuntimeOptions() RuntimeOptions {
 	options := RuntimeOptions{
+		Options: syncthing.Options{
+			AssetDir:    os.Getenv("STGUIASSETS"),
+			NoUpgrade:   os.Getenv("STNOUPGRADE") != "",
+			ProfilerURL: os.Getenv("STPROFILER"),
+		},
 		noRestart:    os.Getenv("STNORESTART") != "",
-		profiler:     os.Getenv("STPROFILER"),
-		assetDir:     os.Getenv("STGUIASSETS"),
 		cpuProfile:   os.Getenv("STCPUPROFILE") != "",
 		stRestarting: os.Getenv("STRESTART") != "",
 		logFlags:     log.Ltime,
@@ -232,7 +209,7 @@ func parseCommandLineOptions() RuntimeOptions {
 	flag.BoolVar(&options.browserOnly, "browser-only", false, "Open GUI in browser")
 	flag.BoolVar(&options.noRestart, "no-restart", options.noRestart, "Disable monitor process, managed restarts and log file writing")
 	flag.BoolVar(&options.resetDatabase, "reset-database", false, "Reset the database, forcing a full rescan and resync")
-	flag.BoolVar(&options.resetDeltaIdxs, "reset-deltas", false, "Reset delta index IDs, forcing a full index exchange")
+	flag.BoolVar(&options.ResetDeltaIdxs, "reset-deltas", false, "Reset delta index IDs, forcing a full index exchange")
 	flag.BoolVar(&options.doUpgrade, "upgrade", false, "Perform upgrade")
 	flag.BoolVar(&options.doUpgradeCheck, "upgrade-check", false, "Check for available upgrade")
 	flag.BoolVar(&options.showVersion, "version", false, "Show version")
@@ -241,7 +218,7 @@ func parseCommandLineOptions() RuntimeOptions {
 	flag.BoolVar(&options.showDeviceId, "device-id", false, "Show the device ID")
 	flag.StringVar(&options.upgradeTo, "upgrade-to", options.upgradeTo, "Force upgrade directly from specified URL")
 	flag.BoolVar(&options.auditEnabled, "audit", false, "Write events to audit file")
-	flag.BoolVar(&options.verbose, "verbose", false, "Print verbose log output")
+	flag.BoolVar(&options.Verbose, "verbose", false, "Print verbose log output")
 	flag.BoolVar(&options.paused, "paused", false, "Start with all devices and folders paused")
 	flag.BoolVar(&options.unpaused, "unpaused", false, "Start with all devices and folders unpaused")
 	flag.StringVar(&options.logFile, "logfile", options.logFile, "Log file name (still always logs to stdout). Cannot be used together with -no-restart/STNORESTART environment variable.")
@@ -264,33 +241,6 @@ func parseCommandLineOptions() RuntimeOptions {
 	return options
 }
 
-// exiter implements api.Controller
-type exiter struct {
-	stop chan int
-}
-
-func (e *exiter) Restart() {
-	l.Infoln("Restarting")
-	e.stop <- exitRestarting
-}
-
-func (e *exiter) Shutdown() {
-	l.Infoln("Shutting down")
-	e.stop <- exitSuccess
-}
-
-func (e *exiter) ExitUpgrading() {
-	l.Infoln("Shutting down after upgrade")
-	e.stop <- exitUpgrading
-}
-
-// waitForExit must be called synchronously.
-func (e *exiter) waitForExit() int {
-	return <-e.stop
-}
-
-var exit = &exiter{make(chan int)}
-
 func main() {
 	options := parseCommandLineOptions()
 	l.SetFlags(options.logFlags)
@@ -310,7 +260,7 @@ func main() {
 	// default location
 	if options.noRestart && (options.logFile != "" && options.logFile != "-") {
 		l.Warnln("-logfile may not be used with -no-restart or STNORESTART")
-		os.Exit(exitError)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 
 	if options.hideConsole {
@@ -324,12 +274,12 @@ func main() {
 			options.confDir, err = filepath.Abs(options.confDir)
 			if err != nil {
 				l.Warnln("Failed to make options path absolute:", err)
-				os.Exit(exitError)
+				os.Exit(syncthing.ExitError.AsInt())
 			}
 		}
 		if err := locations.SetBaseDir(locations.ConfigBaseDir, options.confDir); err != nil {
 			l.Warnln(err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 	}
 
@@ -339,10 +289,10 @@ func main() {
 		options.logFile = locations.Get(locations.LogFile)
 	}
 
-	if options.assetDir == "" {
+	if options.AssetDir == "" {
 		// The asset dir is blank if STGUIASSETS wasn't set, in which case we
 		// should look for extra assets in the default place.
-		options.assetDir = locations.Get(locations.GUIAssets)
+		options.AssetDir = locations.Get(locations.GUIAssets)
 	}
 
 	if options.showVersion {
@@ -367,18 +317,17 @@ func main() {
 		)
 		if err != nil {
 			l.Warnln("Error reading device ID:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 
-		myID = protocol.NewDeviceID(cert.Certificate[0])
-		fmt.Println(myID)
+		fmt.Println(protocol.NewDeviceID(cert.Certificate[0]))
 		return
 	}
 
 	if options.browserOnly {
-		if err := openGUI(); err != nil {
+		if err := openGUI(protocol.EmptyDeviceID); err != nil {
 			l.Warnln("Failed to open web UI:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		return
 	}
@@ -386,7 +335,7 @@ func main() {
 	if options.generateDir != "" {
 		if err := generate(options.generateDir); err != nil {
 			l.Warnln("Failed to generate config and keys:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		return
 	}
@@ -394,14 +343,14 @@ func main() {
 	// Ensure that our home directory exists.
 	if err := ensureDir(locations.GetBaseDir(locations.ConfigBaseDir), 0700); err != nil {
 		l.Warnln("Failure on home directory:", err)
-		os.Exit(exitError)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 
 	if options.upgradeTo != "" {
 		err := upgrade.ToURL(options.upgradeTo)
 		if err != nil {
 			l.Warnln("Error while Upgrading:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		l.Infoln("Upgraded from", options.upgradeTo)
 		return
@@ -421,7 +370,7 @@ func main() {
 	if options.resetDatabase {
 		if err := resetDB(); err != nil {
 			l.Warnln("Resetting database:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		return
 	}
@@ -433,8 +382,8 @@ func main() {
 	}
 }
 
-func openGUI() error {
-	cfg, err := loadOrDefaultConfig()
+func openGUI(myID protocol.DeviceID) error {
+	cfg, err := loadOrDefaultConfig(myID, events.NoopLogger)
 	if err != nil {
 		return err
 	}
@@ -458,31 +407,26 @@ func generate(generateDir string) error {
 		return err
 	}
 
+	var myID protocol.DeviceID
 	certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem")
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err == nil {
 		l.Warnln("Key exists; will not overwrite.")
-		l.Infoln("Device ID:", protocol.NewDeviceID(cert.Certificate[0]))
 	} else {
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, tlsDefaultCommonName)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, tlsDefaultCommonName, deviceCertLifetimeDays)
 		if err != nil {
 			return errors.Wrap(err, "create certificate")
 		}
-		myID = protocol.NewDeviceID(cert.Certificate[0])
-		if err != nil {
-			return errors.Wrap(err, "load certificate")
-		}
-		if err == nil {
-			l.Infoln("Device ID:", protocol.NewDeviceID(cert.Certificate[0]))
-		}
 	}
+	myID = protocol.NewDeviceID(cert.Certificate[0])
+	l.Infoln("Device ID:", myID)
 
 	cfgFile := filepath.Join(dir, "config.xml")
 	if _, err := os.Stat(cfgFile); err == nil {
 		l.Warnln("Config exists; will not overwrite.")
 		return nil
 	}
-	cfg, err := defaultConfig(cfgFile)
+	cfg, err := syncthing.DefaultConfig(cfgFile, myID, events.NoopLogger, noDefaultFolder)
 	if err != nil {
 		return err
 	}
@@ -516,18 +460,18 @@ func debugFacilities() string {
 }
 
 func checkUpgrade() upgrade.Release {
-	cfg, _ := loadOrDefaultConfig()
+	cfg, _ := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger)
 	opts := cfg.Options()
 	release, err := upgrade.LatestRelease(opts.ReleasesURL, build.Version, opts.UpgradeToPreReleases)
 	if err != nil {
 		l.Warnln("Upgrade:", err)
-		os.Exit(exitError)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 
 	if upgrade.CompareVersions(release.Tag, build.Version) <= 0 {
 		noUpgradeMessage := "No upgrade available (current %q >= latest %q)."
 		l.Infof(noUpgradeMessage, build.Version, release.Tag)
-		os.Exit(exitNoUpgradeAvailable)
+		os.Exit(syncthing.ExitNoUpgradeAvailable.AsInt())
 	}
 
 	l.Infof("Upgrade available (current %q < latest %q)", build.Version, release.Tag)
@@ -536,12 +480,12 @@ func checkUpgrade() upgrade.Release {
 
 func performUpgrade(release upgrade.Release) {
 	// Use leveldb database locks to protect against concurrent upgrades
-	_, err := db.Open(locations.Get(locations.Database))
+	_, err := syncthing.OpenGoleveldb(locations.Get(locations.Database), config.TuningAuto)
 	if err == nil {
 		err = upgrade.To(release)
 		if err != nil {
 			l.Warnln("Upgrade:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		l.Infof("Upgraded to %q", release.Tag)
 	} else {
@@ -549,15 +493,15 @@ func performUpgrade(release upgrade.Release) {
 		err = upgradeViaRest()
 		if err != nil {
 			l.Warnln("Upgrade:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		l.Infoln("Syncthing upgrading")
-		os.Exit(exitUpgrading)
+		os.Exit(syncthing.ExitUpgrade.AsInt())
 	}
 }
 
 func upgradeViaRest() error {
-	cfg, _ := loadOrDefaultConfig()
+	cfg, _ := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger)
 	u, err := url.Parse(cfg.GUI().URL())
 	if err != nil {
 		return err
@@ -593,174 +537,32 @@ func upgradeViaRest() error {
 }
 
 func syncthingMain(runtimeOptions RuntimeOptions) {
-	setupSignalHandling()
-
-	// Create a main service manager. We'll add things to this as we go along.
-	// We want any logging it does to go through our log system.
-	mainService := suture.New("main", suture.Spec{
-		Log: func(line string) {
-			l.Debugln(line)
-		},
-		PassThroughPanics: true,
-	})
-	mainService.ServeBackground()
-
 	// Set a log prefix similar to the ID we will have later on, or early log
 	// lines look ugly.
 	l.SetPrefix("[start] ")
 
-	if runtimeOptions.auditEnabled {
-		startAuditing(mainService, runtimeOptions.auditFile)
-	}
-
-	if runtimeOptions.verbose {
-		mainService.Add(newVerboseService())
-	}
-
-	errors := logger.NewRecorder(l, logger.LevelWarn, maxSystemErrors, 0)
-	systemLog := logger.NewRecorder(l, logger.LevelDebug, maxSystemLog, initialSystemLog)
-
-	// Event subscription for the API; must start early to catch the early
-	// events. The LocalChangeDetected event might overwhelm the event
-	// receiver in some situations so we will not subscribe to it here.
-	defaultSub := events.NewBufferedSubscription(events.Default.Subscribe(api.DefaultEventMask), api.EventSubBufferSize)
-	diskSub := events.NewBufferedSubscription(events.Default.Subscribe(api.DiskEventMask), api.EventSubBufferSize)
-
-	if len(os.Getenv("GOMAXPROCS")) == 0 {
-		runtime.GOMAXPROCS(runtime.NumCPU())
-	}
-
-	// Attempt to increase the limit on number of open files to the maximum
-	// allowed, in case we have many peers. We don't really care enough to
-	// report the error if there is one.
-	osutil.MaximizeOpenFileLimit()
+	// Print our version information up front, so any crash that happens
+	// early etc. will have it available.
+	l.Infoln(build.LongVersion)
 
 	// Ensure that we have a certificate and key.
-	cert, err := tls.LoadX509KeyPair(
+	cert, err := syncthing.LoadOrGenerateCertificate(
 		locations.Get(locations.CertFile),
 		locations.Get(locations.KeyFile),
 	)
 	if err != nil {
-		l.Infof("Generating ECDSA key and certificate for %s...", tlsDefaultCommonName)
-		cert, err = tlsutil.NewCertificate(
-			locations.Get(locations.CertFile),
-			locations.Get(locations.KeyFile),
-			tlsDefaultCommonName,
-		)
-		if err != nil {
-			l.Infoln("Failed to generate certificate:", err)
-			os.Exit(exitError)
-		}
+		l.Warnln("Failed to load/generate certificate:", err)
+		os.Exit(1)
 	}
 
-	myID = protocol.NewDeviceID(cert.Certificate[0])
-	l.SetPrefix(fmt.Sprintf("[%s] ", myID.String()[:5]))
+	evLogger := events.NewLogger()
+	go evLogger.Serve()
+	defer evLogger.Stop()
 
-	l.Infoln(build.LongVersion)
-	l.Infoln("My ID:", myID)
-
-	// Select SHA256 implementation and report. Affected by the
-	// STHASHING environment variable.
-	sha256.SelectAlgo()
-	sha256.Report()
-
-	// Emit the Starting event, now that we know who we are.
-
-	events.Default.Log(events.Starting, map[string]string{
-		"home": locations.GetBaseDir(locations.ConfigBaseDir),
-		"myID": myID.String(),
-	})
-
-	cfg, err := loadConfigAtStartup(runtimeOptions.allowNewerConfig)
+	cfg, err := syncthing.LoadConfigAtStartup(locations.Get(locations.ConfigFile), cert, evLogger, runtimeOptions.allowNewerConfig, noDefaultFolder)
 	if err != nil {
 		l.Warnln("Failed to initialize config:", err)
-		os.Exit(exitError)
-	}
-
-	if err := checkShortIDs(cfg); err != nil {
-		l.Warnln("Short device IDs are in conflict. Unlucky!\n  Regenerate the device ID of one of the following:\n  ", err)
-		os.Exit(exitError)
-	}
-
-	if len(runtimeOptions.profiler) > 0 {
-		go func() {
-			l.Debugln("Starting profiler on", runtimeOptions.profiler)
-			runtime.SetBlockProfileRate(1)
-			err := http.ListenAndServe(runtimeOptions.profiler, nil)
-			if err != nil {
-				l.Warnln(err)
-				os.Exit(exitError)
-			}
-		}()
-	}
-
-	perf := ur.CpuBench(3, 150*time.Millisecond, true)
-	l.Infof("Hashing performance is %.02f MB/s", perf)
-
-	dbFile := locations.Get(locations.Database)
-	ldb, err := db.Open(dbFile)
-	if err != nil {
-		l.Warnln("Error opening database:", err)
-		os.Exit(exitError)
-	}
-	if err := db.UpdateSchema(ldb); err != nil {
-		l.Warnln("Database schema:", err)
-		os.Exit(exitError)
-	}
-
-	if runtimeOptions.resetDeltaIdxs {
-		l.Infoln("Reinitializing delta index IDs")
-		db.DropDeltaIndexIDs(ldb)
-	}
-
-	protectedFiles := []string{
-		locations.Get(locations.Database),
-		locations.Get(locations.ConfigFile),
-		locations.Get(locations.CertFile),
-		locations.Get(locations.KeyFile),
-	}
-
-	// Remove database entries for folders that no longer exist in the config
-	folders := cfg.Folders()
-	for _, folder := range ldb.ListFolders() {
-		if _, ok := folders[folder]; !ok {
-			l.Infof("Cleaning data for dropped folder %q", folder)
-			db.DropFolder(ldb, folder)
-		}
-	}
-
-	// Grab the previously running version string from the database.
-
-	miscDB := db.NewMiscDataNamespace(ldb)
-	prevVersion, _ := miscDB.String("prevVersion")
-
-	// Strip away prerelease/beta stuff and just compare the release
-	// numbers. 0.14.44 to 0.14.45-banana is an upgrade, 0.14.45-banana to
-	// 0.14.45-pineapple is not.
-
-	prevParts := strings.Split(prevVersion, "-")
-	curParts := strings.Split(build.Version, "-")
-	if prevParts[0] != curParts[0] {
-		if prevVersion != "" {
-			l.Infoln("Detected upgrade from", prevVersion, "to", build.Version)
-		}
-
-		// Drop delta indexes in case we've changed random stuff we
-		// shouldn't have. We will resend our index on next connect.
-		db.DropDeltaIndexIDs(ldb)
-
-		// Remember the new version.
-		miscDB.PutString("prevVersion", build.Version)
-	}
-
-	m := model.NewModel(cfg, myID, "syncthing", build.Version, ldb, protectedFiles)
-
-	if t := os.Getenv("STDEADLOCKTIMEOUT"); t != "" {
-		if secs, _ := strconv.Atoi(t); secs > 0 {
-			m.StartDeadlockDetector(time.Duration(secs) * time.Second)
-		}
-	} else if !build.IsRelease || build.IsBeta {
-		m.StartDeadlockDetector(20 * time.Minute)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 
 	if runtimeOptions.unpaused {
@@ -769,126 +571,51 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		setPauseState(cfg, true)
 	}
 
-	// Add and start folders
-	for _, folderCfg := range cfg.Folders() {
-		if folderCfg.Paused {
-			folderCfg.CreateRoot()
-			continue
-		}
-		m.AddFolder(folderCfg)
-		m.StartFolder(folderCfg.ID)
+	dbFile := locations.Get(locations.Database)
+	ldb, err := syncthing.OpenGoleveldb(dbFile, cfg.Options().DatabaseTuning)
+	if err != nil {
+		l.Warnln("Error opening database:", err)
+		os.Exit(1)
 	}
 
-	mainService.Add(m)
-
-	// Start discovery
-
-	cachedDiscovery := discover.NewCachingMux()
-	mainService.Add(cachedDiscovery)
-
-	// The TLS configuration is used for both the listening socket and outgoing
-	// connections.
-
-	tlsCfg := tlsutil.SecureDefault()
-	tlsCfg.Certificates = []tls.Certificate{cert}
-	tlsCfg.NextProtos = []string{bepProtocolName}
-	tlsCfg.ClientAuth = tls.RequestClientCert
-	tlsCfg.SessionTicketsDisabled = true
-	tlsCfg.InsecureSkipVerify = true
-
-	// Start connection management
-
-	connectionsService := connections.NewService(cfg, myID, m, tlsCfg, cachedDiscovery, bepProtocolName, tlsDefaultCommonName)
-	mainService.Add(connectionsService)
-
-	if cfg.Options().GlobalAnnEnabled {
-		for _, srv := range cfg.GlobalDiscoveryServers() {
-			l.Infoln("Using discovery server", srv)
-			gd, err := discover.NewGlobal(srv, cert, connectionsService)
-			if err != nil {
-				l.Warnln("Global discovery:", err)
-				continue
-			}
-
-			// Each global discovery server gets its results cached for five
-			// minutes, and is not asked again for a minute when it's returned
-			// unsuccessfully.
-			cachedDiscovery.Add(gd, 5*time.Minute, time.Minute)
-		}
+	appOpts := runtimeOptions.Options
+	if runtimeOptions.auditEnabled {
+		appOpts.AuditWriter = auditWriter(runtimeOptions.auditFile)
+	}
+	if t := os.Getenv("STDEADLOCKTIMEOUT"); t != "" {
+		secs, _ := strconv.Atoi(t)
+		appOpts.DeadlockTimeoutS = secs
 	}
 
-	if cfg.Options().LocalAnnEnabled {
-		// v4 broadcasts
-		bcd, err := discover.NewLocal(myID, fmt.Sprintf(":%d", cfg.Options().LocalAnnPort), connectionsService)
-		if err != nil {
-			l.Warnln("IPv4 local discovery:", err)
-		} else {
-			cachedDiscovery.Add(bcd, 0, 0)
-		}
-		// v6 multicasts
-		mcd, err := discover.NewLocal(myID, cfg.Options().LocalAnnMCAddr, connectionsService)
-		if err != nil {
-			l.Warnln("IPv6 local discovery:", err)
-		} else {
-			cachedDiscovery.Add(mcd, 0, 0)
-		}
+	app := syncthing.New(cfg, ldb, evLogger, cert, appOpts)
+
+	setupSignalHandling(app)
+
+	if len(os.Getenv("GOMAXPROCS")) == 0 {
+		runtime.GOMAXPROCS(runtime.NumCPU())
 	}
 
 	if runtimeOptions.cpuProfile {
 		f, err := os.Create(fmt.Sprintf("cpu-%d.pprof", os.Getpid()))
 		if err != nil {
 			l.Warnln("Creating profile:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		if err := pprof.StartCPUProfile(f); err != nil {
 			l.Warnln("Starting profile:", err)
-			os.Exit(exitError)
-		}
-	}
-
-	// Candidate builds always run with usage reporting.
-
-	if opts := cfg.Options(); build.IsCandidate {
-		l.Infoln("Anonymous usage reporting is always enabled for candidate releases.")
-		if opts.URAccepted != ur.Version {
-			opts.URAccepted = ur.Version
-			cfg.SetOptions(opts)
-			cfg.Save()
-			// Unique ID will be set and config saved below if necessary.
-		}
-	}
-
-	// If we are going to do usage reporting, ensure we have a valid unique ID.
-	if opts := cfg.Options(); opts.URAccepted > 0 && opts.URUniqueID == "" {
-		opts.URUniqueID = rand.String(8)
-		cfg.SetOptions(opts)
-		cfg.Save()
-	}
-
-	usageReportingSvc := ur.New(cfg, m, connectionsService, noUpgradeFromEnv)
-	mainService.Add(usageReportingSvc)
-
-	// GUI
-
-	setupGUI(mainService, cfg, m, defaultSub, diskSub, cachedDiscovery, connectionsService, usageReportingSvc, errors, systemLog, runtimeOptions)
-
-	myDev, _ := cfg.Device(myID)
-	l.Infof(`My name is "%v"`, myDev.Name)
-	for _, device := range cfg.Devices() {
-		if device.DeviceID != myID {
-			l.Infof(`Device %s is "%v" at %v`, device.DeviceID, device.Name, device.Addresses)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 	}
 
 	if opts := cfg.Options(); opts.RestartOnWakeup {
-		go standbyMonitor()
+		go standbyMonitor(app)
 	}
 
 	// Candidate builds should auto upgrade. Make sure the option is set,
 	// unless we are in a build where it's disabled or the STNOUPGRADE
 	// environment variable is set.
 
-	if build.IsCandidate && !upgrade.DisabledByCompilation && !noUpgradeFromEnv {
+	if build.IsCandidate && !upgrade.DisabledByCompilation && !runtimeOptions.NoUpgrade {
 		l.Infoln("Automatic upgrade is always enabled for candidate releases.")
 		if opts := cfg.Options(); opts.AutoUpgradeIntervalH == 0 || opts.AutoUpgradeIntervalH > 24 {
 			opts.AutoUpgradeIntervalH = 12
@@ -902,54 +629,35 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	}
 
 	if opts := cfg.Options(); opts.AutoUpgradeIntervalH > 0 {
-		if noUpgradeFromEnv {
+		if runtimeOptions.NoUpgrade {
 			l.Infof("No automatic upgrades; STNOUPGRADE environment variable defined.")
 		} else {
-			go autoUpgrade(cfg)
+			go autoUpgrade(cfg, app, evLogger)
 		}
 	}
 
-	if isSuperUser() {
-		l.Warnln("Syncthing should not run as a privileged or system user. Please consider using a normal user account.")
+	if err := app.Start(); err != nil {
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 
-	events.Default.Log(events.StartupComplete, map[string]string{
-		"myID": myID.String(),
-	})
-
 	cleanConfigDirectory()
 
-	if cfg.Options().SetLowPriority {
-		if err := osutil.SetLowPriority(); err != nil {
-			l.Warnln("Failed to lower process priority:", err)
-		}
+	if cfg.Options().StartBrowser && !runtimeOptions.noBrowser && !runtimeOptions.stRestarting {
+		// Can potentially block if the utility we are invoking doesn't
+		// fork, and just execs, hence keep it in its own routine.
+		go func() { _ = openURL(cfg.GUI().URL()) }()
 	}
 
-	code := exit.waitForExit()
-
-	mainService.Stop()
-
-	done := make(chan struct{})
-	go func() {
-		ldb.Close()
-		close(done)
-	}()
-	select {
-	case <-done:
-	case <-time.After(10 * time.Second):
-		l.Warnln("Database failed to stop within 10s")
-	}
-
-	l.Infoln("Exiting")
+	status := app.Wait()
 
 	if runtimeOptions.cpuProfile {
 		pprof.StopCPUProfile()
 	}
 
-	os.Exit(code)
+	os.Exit(int(status))
 }
 
-func setupSignalHandling() {
+func setupSignalHandling(app *syncthing.App) {
 	// Exit cleanly with "restarting" code on SIGHUP.
 
 	restartSign := make(chan os.Signal, 1)
@@ -957,7 +665,7 @@ func setupSignalHandling() {
 	signal.Notify(restartSign, sigHup)
 	go func() {
 		<-restartSign
-		exit.Restart()
+		app.Stop(syncthing.ExitRestart)
 	}()
 
 	// Exit with "success" code (no restart) on INT/TERM
@@ -967,85 +675,22 @@ func setupSignalHandling() {
 	signal.Notify(stopSign, os.Interrupt, sigTerm)
 	go func() {
 		<-stopSign
-		exit.Shutdown()
+		app.Stop(syncthing.ExitSuccess)
 	}()
 }
 
-func loadOrDefaultConfig() (config.Wrapper, error) {
+func loadOrDefaultConfig(myID protocol.DeviceID, evLogger events.Logger) (config.Wrapper, error) {
 	cfgFile := locations.Get(locations.ConfigFile)
-	cfg, err := config.Load(cfgFile, myID)
+	cfg, err := config.Load(cfgFile, myID, evLogger)
 
 	if err != nil {
-		cfg, err = defaultConfig(cfgFile)
+		cfg, err = syncthing.DefaultConfig(cfgFile, myID, evLogger, noDefaultFolder)
 	}
 
 	return cfg, err
 }
 
-func loadConfigAtStartup(allowNewerConfig bool) (config.Wrapper, error) {
-	cfgFile := locations.Get(locations.ConfigFile)
-	cfg, err := config.Load(cfgFile, myID)
-	if os.IsNotExist(err) {
-		cfg, err = defaultConfig(cfgFile)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to generate default config")
-		}
-		err = cfg.Save()
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to save default config")
-		}
-		l.Infof("Default config saved. Edit %s to taste (with Syncthing stopped) or use the GUI", cfg.ConfigPath())
-	} else if err == io.EOF {
-		return nil, errors.New("Failed to load config: unexpected end of file. Truncated or empty configuration?")
-	} else if err != nil {
-		return nil, errors.Wrap(err, "failed to load config")
-	}
-
-	if cfg.RawCopy().OriginalVersion != config.CurrentVersion {
-		if cfg.RawCopy().OriginalVersion == config.CurrentVersion+1101 {
-			l.Infof("Now, THAT's what we call a config from the future! Don't worry. As long as you hit that wire with the connecting hook at precisely eighty-eight miles per hour the instant the lightning strikes the tower... everything will be fine.")
-		}
-		if cfg.RawCopy().OriginalVersion > config.CurrentVersion && !allowNewerConfig {
-			return nil, fmt.Errorf("Config file version (%d) is newer than supported version (%d). If this is expected, use -allow-newer-config to override.", cfg.RawCopy().OriginalVersion, config.CurrentVersion)
-		}
-		err = archiveAndSaveConfig(cfg)
-		if err != nil {
-			return nil, errors.Wrap(err, "config archive")
-		}
-	}
-
-	return cfg, nil
-}
-
-func archiveAndSaveConfig(cfg config.Wrapper) error {
-	// Copy the existing config to an archive copy
-	archivePath := cfg.ConfigPath() + fmt.Sprintf(".v%d", cfg.RawCopy().OriginalVersion)
-	l.Infoln("Archiving a copy of old config file format at:", archivePath)
-	if err := copyFile(cfg.ConfigPath(), archivePath); err != nil {
-		return err
-	}
-
-	// Do a regular atomic config sve
-	return cfg.Save()
-}
-
-func copyFile(src, dst string) error {
-	bs, err := ioutil.ReadFile(src)
-	if err != nil {
-		return err
-	}
-
-	if err := ioutil.WriteFile(dst, bs, 0600); err != nil {
-		// Attempt to clean up
-		os.Remove(dst)
-		return err
-	}
-
-	return nil
-}
-
-func startAuditing(mainService *suture.Supervisor, auditFile string) {
-
+func auditWriter(auditFile string) io.Writer {
 	var fd io.Writer
 	var err error
 	var auditDest string
@@ -1067,67 +712,14 @@ func startAuditing(mainService *suture.Supervisor, auditFile string) {
 		fd, err = os.OpenFile(auditFile, auditFlags, 0600)
 		if err != nil {
 			l.Warnln("Audit:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		auditDest = auditFile
 	}
 
-	auditService := newAuditService(fd)
-	mainService.Add(auditService)
-
-	// We wait for the audit service to fully start before we return, to
-	// ensure we capture all events from the start.
-	auditService.WaitForStart()
-
 	l.Infoln("Audit log in", auditDest)
-}
 
-func setupGUI(mainService *suture.Supervisor, cfg config.Wrapper, m model.Model, defaultSub, diskSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService connections.Service, urService *ur.Service, errors, systemLog logger.Recorder, runtimeOptions RuntimeOptions) {
-	guiCfg := cfg.GUI()
-
-	if !guiCfg.Enabled {
-		return
-	}
-
-	if guiCfg.InsecureAdminAccess {
-		l.Warnln("Insecure admin access is enabled.")
-	}
-
-	cpu := newCPUService()
-	mainService.Add(cpu)
-
-	summaryService := model.NewFolderSummaryService(cfg, m, myID)
-	mainService.Add(summaryService)
-
-	apiSvc := api.New(myID, cfg, runtimeOptions.assetDir, tlsDefaultCommonName, m, defaultSub, diskSub, discoverer, connectionsService, urService, summaryService, errors, systemLog, cpu, exit, noUpgradeFromEnv)
-	mainService.Add(apiSvc)
-
-	if err := apiSvc.WaitForStart(); err != nil {
-		l.Warnln("Failed starting API:", err)
-		os.Exit(exitError)
-	}
-
-	if cfg.Options().StartBrowser && !runtimeOptions.noBrowser && !runtimeOptions.stRestarting {
-		// Can potentially block if the utility we are invoking doesn't
-		// fork, and just execs, hence keep it in its own routine.
-		go func() { _ = openURL(guiCfg.URL()) }()
-	}
-}
-
-func defaultConfig(cfgFile string) (config.Wrapper, error) {
-	newCfg, err := config.NewWithFreePorts(myID)
-	if err != nil {
-		return nil, err
-	}
-
-	if noDefaultFolder {
-		l.Infoln("We will skip creation of a default folder on first start since the proper envvar is set")
-		return config.Wrap(cfgFile, newCfg), nil
-	}
-
-	newCfg.Folders = append(newCfg.Folders, config.NewFolderConfiguration(myID, "default", "Default Folder", fs.FilesystemTypeBasic, locations.Get(locations.DefFolder)))
-	l.Infoln("Default folder created and/or linked to new config")
-	return config.Wrap(cfgFile, newCfg), nil
+	return fd
 }
 
 func resetDB() error {
@@ -1157,7 +749,7 @@ func ensureDir(dir string, mode fs.FileMode) error {
 	return nil
 }
 
-func standbyMonitor() {
+func standbyMonitor(app *syncthing.App) {
 	restartDelay := 60 * time.Second
 	now := time.Now()
 	for {
@@ -1170,16 +762,16 @@ func standbyMonitor() {
 			// things a moment to stabilize.
 			time.Sleep(restartDelay)
 
-			exit.Restart()
+			app.Stop(syncthing.ExitRestart)
 			return
 		}
 		now = time.Now()
 	}
 }
 
-func autoUpgrade(cfg config.Wrapper) {
+func autoUpgrade(cfg config.Wrapper, app *syncthing.App, evLogger events.Logger) {
 	timer := time.NewTimer(0)
-	sub := events.Default.Subscribe(events.DeviceConnected)
+	sub := evLogger.Subscribe(events.DeviceConnected)
 	for {
 		select {
 		case event := <-sub.C():
@@ -1201,7 +793,7 @@ func autoUpgrade(cfg config.Wrapper) {
 
 		rel, err := upgrade.LatestRelease(opts.ReleasesURL, build.Version, opts.UpgradeToPreReleases)
 		if err == upgrade.ErrUpgradeUnsupported {
-			events.Default.Unsubscribe(sub)
+			sub.Unsubscribe()
 			return
 		}
 		if err != nil {
@@ -1225,10 +817,10 @@ func autoUpgrade(cfg config.Wrapper) {
 			timer.Reset(checkInterval)
 			continue
 		}
-		events.Default.Unsubscribe(sub)
+		sub.Unsubscribe()
 		l.Warnf("Automatically upgraded to version %q. Restarting in 1 minute.", rel.Tag)
 		time.Sleep(time.Minute)
-		exit.ExitUpgrading()
+		app.Stop(syncthing.ExitUpgrade)
 		return
 	}
 }
@@ -1276,28 +868,13 @@ func cleanConfigDirectory() {
 	}
 }
 
-// checkShortIDs verifies that the configuration won't result in duplicate
-// short ID:s; that is, that the devices in the cluster all have unique
-// initial 64 bits.
-func checkShortIDs(cfg config.Wrapper) error {
-	exists := make(map[protocol.ShortID]protocol.DeviceID)
-	for deviceID := range cfg.Devices() {
-		shortID := deviceID.Short()
-		if otherID, ok := exists[shortID]; ok {
-			return fmt.Errorf("%v in conflict with %v", deviceID, otherID)
-		}
-		exists[shortID] = deviceID
-	}
-	return nil
-}
-
 func showPaths(options RuntimeOptions) {
 	fmt.Printf("Configuration file:\n\t%s\n\n", locations.Get(locations.ConfigFile))
 	fmt.Printf("Database directory:\n\t%s\n\n", locations.Get(locations.Database))
 	fmt.Printf("Device private key & certificate files:\n\t%s\n\t%s\n\n", locations.Get(locations.KeyFile), locations.Get(locations.CertFile))
 	fmt.Printf("HTTPS private key & certificate files:\n\t%s\n\t%s\n\n", locations.Get(locations.HTTPSKeyFile), locations.Get(locations.HTTPSCertFile))
 	fmt.Printf("Log file:\n\t%s\n\n", options.logFile)
-	fmt.Printf("GUI override directory:\n\t%s\n\n", options.assetDir)
+	fmt.Printf("GUI override directory:\n\t%s\n\n", options.AssetDir)
 	fmt.Printf("Default sync folder directory:\n\t%s\n\n", locations.Get(locations.DefFolder))
 }
 
@@ -1311,6 +888,6 @@ func setPauseState(cfg config.Wrapper, paused bool) {
 	}
 	if _, err := cfg.Replace(raw); err != nil {
 		l.Warnln("Cannot adjust paused state:", err)
-		os.Exit(exitError)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 }

cmd/syncthing/main_test.go

@@ -1,38 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"testing"
-
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-func TestShortIDCheck(t *testing.T) {
-	cfg := config.Wrap("/tmp/test", config.Configuration{
-		Devices: []config.DeviceConfiguration{
-			{DeviceID: protocol.DeviceID{8, 16, 24, 32, 40, 48, 56, 0, 0}},
-			{DeviceID: protocol.DeviceID{8, 16, 24, 32, 40, 48, 56, 1, 1}}, // first 56 bits same, differ in the first 64 bits
-		},
-	})
-
-	if err := checkShortIDs(cfg); err != nil {
-		t.Error("Unexpected error:", err)
-	}
-
-	cfg = config.Wrap("/tmp/test", config.Configuration{
-		Devices: []config.DeviceConfiguration{
-			{DeviceID: protocol.DeviceID{8, 16, 24, 32, 40, 48, 56, 64, 0}},
-			{DeviceID: protocol.DeviceID{8, 16, 24, 32, 40, 48, 56, 64, 1}}, // first 64 bits same
-		},
-	})
-
-	if err := checkShortIDs(cfg); err == nil {
-		t.Error("Should have gotten an error")
-	}
-}

cmd/syncthing/monitor.go

@@ -18,9 +18,12 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/syncthing"
 )
 
 var (
@@ -79,7 +82,7 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 
 		if t := time.Since(restarts[0]); t < loopThreshold {
 			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 
 		copy(restarts[0:], restarts[1:])
@@ -101,7 +104,8 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 		l.Infoln("Starting syncthing")
 		err = cmd.Start()
 		if err != nil {
-			panic(err)
+			l.Warnln("Error starting the main Syncthing process:", err)
+			panic("Error starting the main Syncthing process")
 		}
 
 		stdoutMut.Lock()
@@ -147,17 +151,14 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 				// Successful exit indicates an intentional shutdown
 				return
 			} else if exiterr, ok := err.(*exec.ExitError); ok {
-				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
-					switch status.ExitStatus() {
-					case exitUpgrading:
-						// Restart the monitor process to release the .old
-						// binary as part of the upgrade process.
-						l.Infoln("Restarting monitor...")
-						if err = restartMonitor(args); err != nil {
-							l.Warnln("Restart:", err)
-						}
-						return
+				if exiterr.ExitCode() == syncthing.ExitUpgrade.AsInt() {
+					// Restart the monitor process to release the .old
+					// binary as part of the upgrade process.
+					l.Infoln("Restarting monitor...")
+					if err = restartMonitor(args); err != nil {
+						l.Warnln("Restart:", err)
 					}
+					return
 				}
 			}
 		}
@@ -448,7 +449,7 @@ func childEnv() []string {
 // panicUploadMaxWait uploading panics...
 func maybeReportPanics() {
 	// Try to get a config to see if/where panics should be reported.
-	cfg, err := loadOrDefaultConfig()
+	cfg, err := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger)
 	if err != nil {
 		l.Warnln("Couldn't load config; not reporting crash")
 		return

cmd/todos/main.go

@@ -1,39 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build ignore
-
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"os"
-)
-
-func main() {
-	buf := make([]byte, 4096)
-	var err error
-	for err == nil {
-		n, err := io.ReadFull(os.Stdin, buf)
-		if n > 0 {
-			buf = buf[:n]
-			repl := bytes.Replace(buf, []byte("\n"), []byte("\r\n"), -1)
-			_, err = os.Stdout.Write(repl)
-			if err != nil {
-				fmt.Println(err)
-				os.Exit(1)
-			}
-		}
-		if err == io.EOF {
-			return
-		}
-		buf = buf[:cap(buf)]
-	}
-	fmt.Println(err)
-	os.Exit(1)
-}

cmd/ursrv/main.go

@@ -755,6 +755,8 @@ func main() {
 	http.HandleFunc("/blockstats.json", withDB(db, blockStatsHandler))
 	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
 
+	go cacheRefresher(db)
+
 	err = srv.Serve(listener)
 	if err != nil {
 		log.Fatalln("https:", err)
@@ -767,7 +769,31 @@ var (
 	cacheMut  sync.Mutex
 )
 
-const maxCacheTime = 5 * 60 * time.Second
+const maxCacheTime = 15 * time.Minute
+
+func cacheRefresher(db *sql.DB) {
+	ticker := time.NewTicker(maxCacheTime - time.Minute)
+	defer ticker.Stop()
+	for range ticker.C {
+		cacheMut.Lock()
+		if err := refreshCacheLocked(db); err != nil {
+			log.Println(err)
+		}
+		cacheMut.Unlock()
+	}
+}
+
+func refreshCacheLocked(db *sql.DB) error {
+	rep := getReport(db)
+	buf := new(bytes.Buffer)
+	err := tpl.Execute(buf, rep)
+	if err != nil {
+		return err
+	}
+	cacheData = buf.Bytes()
+	cacheTime = time.Now()
+	return nil
+}
 
 func rootHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
 	if r.URL.Path == "/" || r.URL.Path == "/index.html" {
@@ -775,16 +801,11 @@ func rootHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
 		defer cacheMut.Unlock()
 
 		if time.Since(cacheTime) > maxCacheTime {
-			rep := getReport(db)
-			buf := new(bytes.Buffer)
-			err := tpl.Execute(buf, rep)
-			if err != nil {
+			if err := refreshCacheLocked(db); err != nil {
 				log.Println(err)
 				http.Error(w, "Template Error", http.StatusInternalServerError)
 				return
 			}
-			cacheData = buf.Bytes()
-			cacheTime = time.Now()
 		}
 
 		w.Header().Set("Content-Type", "text/html; charset=utf-8")

go.mod

@@ -4,47 +4,50 @@ require (
 	github.com/AudriusButkevicius/go-nat-pmp v0.0.0-20160522074932-452c97607362
 	github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6
 	github.com/AudriusButkevicius/recli v0.0.5
+	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e
-	github.com/calmh/du v1.0.1
 	github.com/calmh/xdr v1.1.0
 	github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d
-	github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 // indirect
+	github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67 // indirect
 	github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5
 	github.com/d4l3k/messagediff v1.2.1
 	github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
 	github.com/getsentry/raven-go v0.2.0
-	github.com/gobwas/glob v0.0.0-20170212200151-51eb1ee00b6d
-	github.com/gogo/protobuf v1.2.1
-	github.com/golang/groupcache v0.0.0-20171101203131-84a468cf14b4
-	github.com/jackpal/gateway v0.0.0-20161225004348-5795ac81146e
-	github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657
+	github.com/go-ole/go-ole v1.2.4 // indirect
+	github.com/gobwas/glob v0.2.3
+	github.com/gogo/protobuf v1.3.1
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6
+	github.com/golang/mock v1.3.1 // indirect
+	github.com/jackpal/gateway v1.0.5
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/kr/pretty v0.1.0 // indirect
-	github.com/lib/pq v1.1.1
-	github.com/lucas-clemente/quic-go v0.11.2
-	github.com/maruel/panicparse v1.2.1
-	github.com/mattn/go-isatty v0.0.7
-	github.com/minio/sha256-simd v0.0.0-20190117184323-cc1980cb0338
-	github.com/onsi/ginkgo v1.8.0 // indirect
-	github.com/onsi/gomega v1.5.0 // indirect
+	github.com/lib/pq v1.2.0
+	github.com/lucas-clemente/quic-go v0.12.1
+	github.com/maruel/panicparse v1.3.0
+	github.com/mattn/go-isatty v0.0.10
+	github.com/minio/sha256-simd v0.1.1
+	github.com/onsi/ginkgo v1.9.0 // indirect
+	github.com/onsi/gomega v1.6.0 // indirect
 	github.com/oschwald/geoip2-golang v1.3.0
-	github.com/oschwald/maxminddb-golang v0.0.0-20170901134056-26fe5ace1c70 // indirect
-	github.com/petermattis/goid v0.0.0-20170816195418-3db12ebb2a59 // indirect
+	github.com/oschwald/maxminddb-golang v1.4.0 // indirect
+	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
 	github.com/pkg/errors v0.8.1
-	github.com/prometheus/client_golang v0.9.4
-	github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9
+	github.com/prometheus/client_golang v1.2.1
+	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563
 	github.com/sasha-s/go-deadlock v0.2.0
-	github.com/syncthing/notify v0.0.0-20181107104724-4e389ea6c0d8
-	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
+	github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf
+	github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2
+	github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d
 	github.com/thejerf/suture v3.0.2+incompatible
-	github.com/urfave/cli v1.20.0
+	github.com/urfave/cli v1.22.1
 	github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0
-	golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8
-	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
-	golang.org/x/sys v0.0.0-20190613124609-5ed2794edfdc // indirect
+	golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472
+	golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297
 	golang.org/x/text v0.3.2
-	golang.org/x/time v0.0.0-20170927054726-6dc17368e09b
-	gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225 // indirect
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
+	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/ldap.v2 v2.5.1
-	gopkg.in/yaml.v2 v2.2.2 // indirect
 )
+
+go 1.12

go.sum

@@ -1,31 +1,40 @@
 github.com/AudriusButkevicius/go-nat-pmp v0.0.0-20160522074932-452c97607362 h1:l4qGIzSY0WhdXdR74XMYAtfc0Ri/RJVM4p6x/E/+WkA=
 github.com/AudriusButkevicius/go-nat-pmp v0.0.0-20160522074932-452c97607362/go.mod h1:CEaBhA5lh1spxbPOELh5wNLKGsVQoahjUhVrJViVK8s=
-github.com/AudriusButkevicius/pfilter v0.0.0-20190525131515-730b0de4d4de h1:w1VG0ehgPh2ucQGO7wL9TBmHLzMo4dduYwyp2lhs8+A=
-github.com/AudriusButkevicius/pfilter v0.0.0-20190525131515-730b0de4d4de/go.mod h1:1N0EEx/irz4B1qV17wW82TFbjQrE7oX316Cki6eDY0Q=
 github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6 h1:Apvc4kyfdrOxG+F5dn8osz+45kwGJa6CySQn0tB38SU=
 github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6/go.mod h1:1N0EEx/irz4B1qV17wW82TFbjQrE7oX316Cki6eDY0Q=
 github.com/AudriusButkevicius/recli v0.0.5 h1:xUa55PvWTHBm17T6RvjElRO3y5tALpdceH86vhzQ5wg=
 github.com/AudriusButkevicius/recli v0.0.5/go.mod h1:Q2E26yc6RvWWEz/TJ/goUp6yXvipYdJI096hpoaqsNs=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e h1:2augTYh6E+XoNrrivZJBadpThP/dsvYKj0nzqfQ8tM4=
 github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
-github.com/calmh/du v1.0.1 h1:uDCrDbXVVPrzxSNRkpj6nqSfwrl5uRWH3zvrJgl7RRo=
-github.com/calmh/du v1.0.1/go.mod h1:pHNccp4cXQeyDaiV3S7t5GN+eGOgynF0VSLxJjk9tLU=
 github.com/calmh/xdr v1.1.0 h1:U/Dd4CXNLoo8EiQ4ulJUXkgO1/EyQLgDKLgpY1SOoJE=
 github.com/calmh/xdr v1.1.0/go.mod h1:E8sz2ByAdXC8MbANf1LCRYzedSnnc+/sXXJs/PVqoeg=
 github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d h1:As4937T5NVbJ/DmZT9z33pyLEprMd6CUSfhbmMY57Io=
 github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d/go.mod h1:3FK1bMar37f7jqVY7q/63k3OMX1c47pGCufzt3X0sYE=
-github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 h1:UNOqI3EKhvbqV8f1Vm3NIwkrhq388sGCeAH2Op7w0rc=
-github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
+github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67 h1:8k9FLYBLKT+9v2HQJ/a95ZemmTx+/ltJcAiRhVushG8=
+github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
+github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
+github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5 h1:Wg96Dh0MLTanEaPO0OkGtUIaa2jOnShAIOVUIzRHUxo=
 github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U=
 github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -38,32 +47,49 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
 github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
+github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gobwas/glob v0.0.0-20170212200151-51eb1ee00b6d h1:IngNQgbqr5ZOU0exk395Szrvkzes9Ilk1fmJfkw7d+M=
-github.com/gobwas/glob v0.0.0-20170212200151-51eb1ee00b6d/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/golang/groupcache v0.0.0-20171101203131-84a468cf14b4 h1:6o8aP0LGMKzo3NzwhhX6EJsiJ3ejmj+9yA/3p8Fjjlw=
-github.com/golang/groupcache v0.0.0-20171101203131-84a468cf14b4/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
+github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/jackpal/gateway v0.0.0-20161225004348-5795ac81146e h1:lS8IitpqG4RkZbEDlZg5Z7FvBdWLVjSVfsPGOKafEkI=
-github.com/jackpal/gateway v0.0.0-20161225004348-5795ac81146e/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
+github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc=
+github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657 h1:vE7J1m7cCpiRVEIr1B5ccDxRpbPsWT5JU3if2Di5nE4=
-github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -72,94 +98,136 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
-github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lucas-clemente/quic-go v0.11.2 h1:Mop0ac3zALaBR3wGs6j8OYe/tcFvFsxTUFMkE/7yUOI=
-github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
-github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA=
-github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
-github.com/maruel/panicparse v1.2.1 h1:mNlHGiakrixj+AwF/qRpTwnj+zsWYPRLQ7wRqnJsfO0=
-github.com/maruel/panicparse v1.2.1/go.mod h1:vszMjr5QQ4F5FSRfraldcIA/BCw5xrdLL+zEcU2nRBs=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lucas-clemente/quic-go v0.12.0 h1:TRbvZ6F++sofeGbh+Z2IIyIOhl8KyGnYuA06g2yrHdI=
+github.com/lucas-clemente/quic-go v0.12.0/go.mod h1:UXJJPE4RfFef/xPO5wQm0tITK8gNfqwTxjbE7s3Vb8s=
+github.com/lucas-clemente/quic-go v0.12.1 h1:BPITli+6KnKogtTxBk2aS4okr5dUHz2LtIDAP1b8UL4=
+github.com/lucas-clemente/quic-go v0.12.1/go.mod h1:UXJJPE4RfFef/xPO5wQm0tITK8gNfqwTxjbE7s3Vb8s=
+github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
+github.com/marten-seemann/qtls v0.3.2 h1:O7awy4bHEzSX/K3h+fZig3/Vo03s/RxlxgsAk9sYamI=
+github.com/marten-seemann/qtls v0.3.2/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
+github.com/maruel/panicparse v1.3.0 h1:1Ep/RaYoSL1r5rTILHQQbyzHG8T4UP5ZbQTYTo4bdDc=
+github.com/maruel/panicparse v1.3.0/go.mod h1:vszMjr5QQ4F5FSRfraldcIA/BCw5xrdLL+zEcU2nRBs=
+github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
-github.com/minio/sha256-simd v0.0.0-20190117184323-cc1980cb0338 h1:USW1+zAUkUSvk097CAX/i8KR3r6f+DHNhk6Xe025Oyw=
-github.com/minio/sha256-simd v0.0.0-20190117184323-cc1980cb0338/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc=
+github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M=
+github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/oschwald/geoip2-golang v1.3.0 h1:D+Hsdos1NARPbzZ2aInUHZL+dApIzo8E0ErJVsWcku8=
 github.com/oschwald/geoip2-golang v1.3.0/go.mod h1:0LTTzix/Ao1uMvOhAV4iLU0Lz7eCrP94qZWBTDKf0iE=
-github.com/oschwald/maxminddb-golang v0.0.0-20170901134056-26fe5ace1c70 h1:XGLYUmodtNzThosQ8GkMvj9TiIB/uWsP8NfxKSa3aDc=
-github.com/oschwald/maxminddb-golang v0.0.0-20170901134056-26fe5ace1c70/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
-github.com/petermattis/goid v0.0.0-20170816195418-3db12ebb2a59 h1:2pHcLyJYXivxVvpoCc29uo3GDU1qFfJ1ggXKGYMrM0E=
-github.com/petermattis/goid v0.0.0-20170816195418-3db12ebb2a59/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
+github.com/oschwald/maxminddb-golang v1.4.0 h1:5/rpmW41qrgSed4wK32rdznbkTSXHcraY2LOMJX4DMc=
+github.com/oschwald/maxminddb-golang v1.4.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
+github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ=
+github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.4 h1:Y8E/JaaPbmFSW2V81Ab/d8yZFYQQGbni1b1jPcG9Y6A=
-github.com/prometheus/client_golang v0.9.4/go.mod h1:oCXIBxdI62A4cR6aTRJCgetEjecSIYzOEaeAn4iYEpM=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8=
+github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI=
+github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo=
+github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9 h1:jmLW6izPBVlIbk4d+XgK9+sChGbVKxxOPmd9eqRHCjw=
-github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
+github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.0.4 h1:w8DjqFMJDjuVwdZBQoOozr4MVWOnwF7RcL/7uxBjY78=
+github.com/prometheus/procfs v0.0.4/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
+github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
+github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y=
 github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
+github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf h1:c9SV5NzG4KOk448TUE7iqCmb4E4y79CZF4zDdc1Jx3Q=
+github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/syncthing/notify v0.0.0-20181107104724-4e389ea6c0d8 h1:ewsMW/a4xDpqHyIteoD29ayMn6GdkFZc2T0PX2K6PAg=
-github.com/syncthing/notify v0.0.0-20181107104724-4e389ea6c0d8/go.mod h1:Sn4ChoS7e4FxjCN1XHPVBT43AgnRLbuaB8pEc1Zcdjg=
+github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2 h1:6tuEEEpg+mxM82E0YingzoXzXXISYR/o/7I9n573LWI=
+github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2/go.mod h1:Sn4ChoS7e4FxjCN1XHPVBT43AgnRLbuaB8pEc1Zcdjg=
 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
+github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs=
+github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
 github.com/thejerf/suture v3.0.2+incompatible h1:GtMydYcnK4zBJ0KL6Lx9vLzl6Oozb65wh252FTBxrvM=
 github.com/thejerf/suture v3.0.2+incompatible/go.mod h1:ibKwrVj+Uzf3XZdAiNWUouPaAbSoemxOHLmJmwheEMc=
 github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
+github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0 h1:okhMind4q9H1OxF44gNegWkiP4H/gsTFLalHFa4OOUI=
 github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0/go.mod h1:TTbGUfE+cXXceWtbTHq6lqcTvYPBKLNejBEbnUsQJtU=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM=
+golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -171,19 +239,29 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e h1:ZytStCyV048ZqDsWHiYDdoI2Vd4msMcrDECFxS+tL9c=
 golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190613124609-5ed2794edfdc h1:x+/QxSNkVFAC+v4pL1f6mZr1z+qgi+FoR8ccXZPVC10=
-golang.org/x/sys v0.0.0-20190613124609-5ed2794edfdc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/time v0.0.0-20170927054726-6dc17368e09b h1:3X+R0qq1+64izd8es+EttB6qcY+JDlVmAhpRXl7gpzU=
-golang.org/x/time v0.0.0-20170927054726-6dc17368e09b/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
-gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225 h1:JBwmEvLfCqgPcIq8MjVMQxsF3LVL4XG/HH0qiG0+IFY=
-gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
+gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
+gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

gui/black/assets/css/theme.css

@@ -24,6 +24,27 @@ a:hover,a:focus,a.focus{
     border-width: 2px !important;
 }
 
+
+.nav-tabs > li.active > a,
+.nav-tabs > li.active > a:hover,
+.nav-tabs > li.active > a:focus {
+  color: #3498db !important;
+  background-color: #222222 !important;
+  border: 1px solid #222222 !important;
+  border-bottom-color: transparent !important;
+  cursor: default;
+}
+
+.nav-tabs{
+    border-bottom: 1px solid #333;
+}
+
+.nav-tabs > li > a:hover,
+.nav-tabs > li > a:focus {
+  background-color: #222222 !important;
+  border: none !important;
+}
+
 .navbar-text, .dropdown>a, .dropdown-menu>li>a, .hidden-xs>a, .navbar-link {
     color: #aaa !important;
 }

gui/dark/assets/css/theme.css

@@ -28,6 +28,27 @@ a:hover,a:focus,a.focus{
 
 }
 
+.nav-tabs > li.active > a,
+.nav-tabs > li.active > a:hover,
+.nav-tabs > li.active > a:focus {
+  color: #3498db !important;
+  background-color: #424242 !important;
+  border: 1px solid #424242 !important;
+  border-bottom-color: transparent !important;
+  cursor: default;
+}
+
+.nav-tabs{
+    border-bottom: 1px solid #333;
+}
+
+.nav-tabs > li > a:hover,
+.nav-tabs > li > a:focus {
+  background-color: #424242 !important;
+  border: none !important;
+}
+
+
 .navbar-text, .dropdown>a, .dropdown-menu>li>a, .hidden-xs>a, .navbar-link {
     color: #aaa !important;
 }

gui/default/assets/css/theme.css

@@ -7,29 +7,5 @@
 
 */
 
-.panel-progress {
-    background: #3498db;
-}
-
-.identicon rect {
-    fill: #333;
-}
-
-.panel-warning .identicon rect {
-    fill: #fff;
-}
-
-.li-column {
-    background-color: rgb(236, 240, 241);
-    border-radius: 3px;
-}
-
-.panel-heading:hover, .panel-heading:focus {
-    text-decoration: none;
-}
-
-.fancytree-ext-filter-hide tr.fancytree-submatch span.fancytree-title,
-.fancytree-ext-filter-hide span.fancytree-node.fancytree-submatch span.fancytree-title {
-    color: black !important;
-    font-weight: lighter !important;
-}
+@import "../../theme-assets/dark/assets/css/theme.css" screen and (prefers-color-scheme: dark);
+@import "../../theme-assets/light/assets/css/theme.css" (prefers-color-scheme: light), (prefers-color-scheme: no-preference);

gui/default/index.html

@@ -323,9 +323,14 @@
                     <span class="visible-xs" aria-label="{{'Scanning' | translate}}"><i class="fas fa-fw fa-search"></i></span>
                   </span>
                   <span ng-switch-when="idle"><span class="hidden-xs" translate>Up to Date</span><span class="visible-xs" aria-label="{{'Up to Date' | translate}}"><i class="fas fa-fw fa-check"></i></span></span>
+                  <span ng-switch-when="localadditions"><span class="hidden-xs" translate>Local Additions</span><span class="visible-xs" aria-label="{{'Local Additions' | translate}}"><i class="fas fa-fw fa-check"></i></span></span>
+                  <span ng-switch-when="sync-preparing">
+                    <span class="hidden-xs" translate>Preparing to Sync</span>
+                    <span class="visible-xs" aria-label="{{'Preparing to Sync' | translate}}"><i class="fas fa-fw fa-hourglass-half"></i></span>
+                  </span>
                   <span ng-switch-when="syncing">
                     <span class="hidden-xs" translate>Syncing</span>
-                    <span ng-show="syncRemaining(folder.id)">({{syncPercentage(folder.id) | percent}}, {{syncRemaining(folder.id) | binary}}B)</span>
+                    <span>({{syncPercentage(folder.id) | percent}}, {{model[folder.id].needBytes | binary}}B)</span>
                   </span>
                   <span ng-switch-when="outofsync"><span class="hidden-xs" translate>Out of Sync</span><span class="visible-xs" aria-label="{{'Out of Sync' | translate}}"><i class="fas fa-fw fa-exclamation-circle"></i></span></span>
                   <span ng-switch-when="faileditems"><span class="hidden-xs" translate>Failed Items</span><span class="visible-xs" aria-label="{{'Failed Items' | translate}}"><i class="fas fa-fw fa-exclamation-circle"></i></span></span>

gui/default/syncthing/app.js

@@ -66,6 +66,14 @@ function folderCompare(a, b) {
     return labelA > labelB;
 }
 
+function deviceMap(l) {
+    var m = {};
+    l.forEach(function (r) {
+        m[r.deviceID] = r;
+    });
+    return m;
+}
+
 function folderMap(l) {
     var m = {};
     l.forEach(function (r) {

gui/default/syncthing/core/syncthingController.js

@@ -60,7 +60,9 @@ angular.module('syncthing.core')
         } catch (exception) { }
 
         $scope.folderDefaults = {
+            sharedDevices: {},
             selectedDevices: {},
+            unrelatedDevices: {},
             type: "sendreceive",
             rescanIntervalS: 3600,
             fsWatcherDelayS: 10,
@@ -386,15 +388,8 @@ angular.module('syncthing.core')
                 });
             });
 
-            // If we're not listening on localhost, and there is no
-            // authentication configured, and the magic setting to silence the
-            // warning isn't set, then yell at the user.
-            var guiCfg = $scope.config.gui;
-            $scope.openNoAuth = guiCfg.address.substr(0, 4) !== "127."
-                && guiCfg.address.substr(0, 6) !== "[::1]:"
-                && (!guiCfg.user || !guiCfg.password)
-                && guiCfg.authMode !== 'ldap'
-                && !guiCfg.insecureAdminAccess;
+            refreshNoAuthWarning();
+            setDefaultTheme();
 
             if (!hasConfig) {
                 $scope.$emit('ConfigLoaded');
@@ -427,10 +422,33 @@ angular.module('syncthing.core')
                     }
                 }
                 $scope.discoveryFailed = discoveryFailed;
+
+                refreshNoAuthWarning();
+
                 console.log("refreshSystem", data);
             }).error($scope.emitHTTPError);
         }
 
+        function refreshNoAuthWarning() {
+            if (!$scope.system || !$scope.config) {
+                // We need both to be able to determine the state.
+                return
+            }
+
+            // If we're not listening on localhost, and there is no
+            // authentication configured, and the magic setting to silence the
+            // warning isn't set, then yell at the user.
+            var addr = $scope.system.guiAddressUsed;
+            var guiCfg = $scope.config.gui;
+            $scope.openNoAuth = addr.substr(0, 4) !== "127."
+                && addr.substr(0, 6) !== "[::1]:"
+                && addr.substr(0, 1) !== "/"
+                && (!guiCfg.user || !guiCfg.password)
+                && guiCfg.authMode !== 'ldap'
+                && !guiCfg.insecureAdminAccess;
+        }
+
+
         function refreshDiscoveryCache() {
             $http.get(urlbase + '/system/discovery').success(function (data) {
                 for (var device in data) {
@@ -634,6 +652,23 @@ angular.module('syncthing.core')
             $scope.remoteNeedDevice = undefined;
         }
 
+
+        function setDefaultTheme() {
+            if (!document.getElementById("fallback-theme-css")){
+
+                // check if no support for prefers-color-scheme
+                var colorSchemeNotSupported = typeof window.matchMedia === "undefined" || window.matchMedia('(prefers-color-scheme: dark)').media === 'not all';
+
+                if ($scope.config.gui.theme === "default" && colorSchemeNotSupported) {
+                    document.documentElement.style.display = 'none';
+                    document.head.insertAdjacentHTML(
+                      'beforeend',
+                      '<link id="fallback-theme-css" rel="stylesheet" href="theme-assets/light/assets/css/theme.css" onload="document.documentElement.style.display = \'\'">'
+                    );
+                }
+            }
+        }
+
         function saveIgnores(ignores, cb) {
             $http.post(urlbase + '/db/ignores?folder=' + encodeURIComponent($scope.currentFolder.id), {
                 ignore: ignores
@@ -753,25 +788,31 @@ angular.module('syncthing.core')
                 return 'paused';
             }
 
+            var folderInfo = $scope.model[folderCfg.id];
+
             // after restart syncthing process state may be empty
-            if (!$scope.model[folderCfg.id].state) {
+            if (!folderInfo.state) {
                 return 'unknown';
             }
 
-            var state = '' + $scope.model[folderCfg.id].state;
+            var state = '' + folderInfo.state;
             if (state === 'error') {
                 return 'stopped'; // legacy, the state is called "stopped" in the GUI
             }
-            if (state === 'idle' && $scope.model[folderCfg.id].needTotalItems > 0) {
+
+            if (state !== 'idle') {
+                return state;
+            }
+
+            if (folderInfo.needTotalItems > 0) {
                 return 'outofsync';
             }
             if ($scope.hasFailedFiles(folderCfg.id)) {
                 return 'faileditems';
             }
-            if (state === 'scanning') {
-                return state;
+            if (folderInfo.receiveOnlyTotalItems) {
+                return 'localadditions';
             }
-
             if (folderCfg.devices.length <= 1) {
                 return 'unshared';
             }
@@ -782,13 +823,13 @@ angular.module('syncthing.core')
         $scope.folderClass = function (folderCfg) {
             var status = $scope.folderStatus(folderCfg);
 
-            if (status === 'idle') {
+            if (status === 'idle' || status === 'localadditions') {
                 return 'success';
             }
             if (status == 'paused') {
                 return 'default';
             }
-            if (status === 'syncing' || status === 'scanning') {
+            if (status === 'syncing' || status === 'sync-preparing' || status === 'scanning') {
                 return 'primary';
             }
             if (status === 'unknown') {
@@ -816,22 +857,6 @@ angular.module('syncthing.core')
             return Math.floor(pct);
         };
 
-        $scope.syncRemaining = function (folder) {
-            // Remaining sync bytes
-            if (typeof $scope.model[folder] === 'undefined') {
-                return 0;
-            }
-            if ($scope.model[folder].globalBytes === 0) {
-                return 0;
-            }
-
-            var bytes = $scope.model[folder].globalBytes - $scope.model[folder].inSyncBytes;
-            if (isNaN(bytes) || bytes < 0) {
-                return 0;
-            }
-            return bytes;
-        };
-
         $scope.scanPercentage = function (folder) {
             if (!$scope.scanProgress[folder]) {
                 return undefined;
@@ -854,6 +879,9 @@ angular.module('syncthing.core')
             // 32m 40s
             // 2h 32m
             // 4d 2h
+            // In case remaining scan time appears to be >31d, omit the
+            // details, i.e.:
+            // > 1 month
 
             if (!$scope.scanProgress[folder]) {
                 return "";
@@ -873,6 +901,9 @@ angular.module('syncthing.core')
             var res = [];
             if (seconds >= 86400) {
                 days = Math.floor(seconds / 86400);
+                if (days > 31) {
+                    return '> 1 month';
+                }
                 res.push('' + days + 'd')
                 seconds = seconds % 86400;
             }
@@ -963,6 +994,7 @@ angular.module('syncthing.core')
             for (var i = 0; i < folderListCache.length; i++) {
                 var status = $scope.folderStatus(folderListCache[i]);
                 switch (status) {
+                    case 'sync-preparing':
                     case 'syncing':
                         syncCount++;
                         break;
@@ -1295,6 +1327,13 @@ angular.module('syncthing.core')
                     $scope.protocolChanged = true;
                 }
 
+                // Parse strings to arrays before copying over
+                ['listenAddresses', 'globalAnnounceServers'].forEach(function (key) {
+                    $scope.tmpOptions[key] = $scope.tmpOptions["_" + key + "Str"].split(/[ ,]+/).map(function (x) {
+                        return x.trim();
+                    });
+                });
+
                 // Apply new settings locally
                 $scope.thisDeviceIn($scope.tmpDevices).name = $scope.tmpOptions.deviceName;
                 $scope.config.options = angular.copy($scope.tmpOptions);
@@ -1308,12 +1347,6 @@ angular.module('syncthing.core')
                 // here as well...
                 $scope.devices = $scope.config.devices;
 
-                ['listenAddresses', 'globalAnnounceServers'].forEach(function (key) {
-                    $scope.config.options[key] = $scope.config.options["_" + key + "Str"].split(/[ ,]+/).map(function (x) {
-                        return x.trim();
-                    });
-                });
-
                 $scope.saveConfig(function () {
                     if (themeChanged) {
                         document.location.reload(true);
@@ -1392,13 +1425,13 @@ angular.module('syncthing.core')
         };
 
         $scope.selectAllFolders = function () {
-            angular.forEach($scope.folders, function (id) {
+            angular.forEach($scope.folders, function (_, id) {
                 $scope.currentDevice.selectedFolders[id] = true;
             });
         };
 
         $scope.deSelectAllFolders = function () {
-            angular.forEach($scope.folders, function (id) {
+            angular.forEach($scope.folders, function (_, id) {
                 $scope.currentDevice.selectedFolders[id] = false;
             });
         };
@@ -1672,10 +1705,20 @@ angular.module('syncthing.core')
             if ($scope.currentFolder.path.length > 1 && $scope.currentFolder.path.slice(-1) === $scope.system.pathSeparator) {
                 $scope.currentFolder.path = $scope.currentFolder.path.slice(0, -1);
             }
+            // Cache complete device objects indexed by ID for lookups
+            var devMap = deviceMap($scope.devices)
+            $scope.currentFolder.sharedDevices = [];
             $scope.currentFolder.selectedDevices = {};
             $scope.currentFolder.devices.forEach(function (n) {
+                if (n.deviceID !== $scope.myID) {
+                    $scope.currentFolder.sharedDevices.push(devMap[n.deviceID]);
+                }
                 $scope.currentFolder.selectedDevices[n.deviceID] = true;
             });
+            $scope.currentFolder.unrelatedDevices = $scope.devices.filter(function (n) {
+                return n.deviceID !== $scope.myID
+                    && ! $scope.currentFolder.selectedDevices[n.deviceID]
+            });
             if ($scope.currentFolder.versioning && $scope.currentFolder.versioning.type === "trashcan") {
                 $scope.currentFolder.trashcanFileVersioning = true;
                 $scope.currentFolder.fileVersioningSelector = "trashcan";
@@ -1726,17 +1769,17 @@ angular.module('syncthing.core')
             $scope.editFolderModal();
         };
 
-        $scope.selectAllDevices = function () {
-            var devices = $scope.otherDevices();
+        $scope.selectAllSharedDevices = function (state) {
+            var devices = $scope.currentFolder.sharedDevices;
             for (var i = 0; i < devices.length; i++) {
-                $scope.currentFolder.selectedDevices[devices[i].deviceID] = true;
+                $scope.currentFolder.selectedDevices[devices[i].deviceID] = !!state;
             }
         };
 
-        $scope.deSelectAllDevices = function () {
-            var devices = $scope.otherDevices();
+        $scope.selectAllUnrelatedDevices = function (state) {
+            var devices = $scope.currentFolder.unrelatedDevices;
             for (var i = 0; i < devices.length; i++) {
-                $scope.currentFolder.selectedDevices[devices[i].deviceID] = false;
+                $scope.currentFolder.selectedDevices[devices[i].deviceID] = !!state;
             }
         };
 
@@ -1745,6 +1788,7 @@ angular.module('syncthing.core')
                 $scope.editingExisting = false;
                 $scope.currentFolder = angular.copy($scope.folderDefaults);
                 $scope.currentFolder.id = (data.random.substr(0, 5) + '-' + data.random.substr(5, 5)).toLowerCase();
+                $scope.currentFolder.unrelatedDevices = $scope.otherDevices();
                 $('#folder-ignores textarea').val("");
                 $('#folder-ignores textarea').removeAttr('disabled');
                 $scope.editFolderModal();
@@ -1760,6 +1804,7 @@ angular.module('syncthing.core')
                 importFromOtherDevice: true
             };
             $scope.currentFolder.selectedDevices[device] = true;
+            $scope.currentFolder.unrelatedDevices = $scope.otherDevices();
             $('#folder-ignores textarea').val("");
             $('#folder-ignores textarea').removeAttr('disabled');
             $scope.editFolderModal();
@@ -1785,7 +1830,9 @@ angular.module('syncthing.core')
                     });
                 }
             }
+            delete folderCfg.sharedDevices;
             delete folderCfg.selectedDevices;
+            delete folderCfg.unrelatedDevices;
 
             if (folderCfg.fileVersioningSelector === "trashcan") {
                 folderCfg.versioning = {

gui/default/syncthing/folder/editFolderModalView.html

@@ -44,15 +44,35 @@
           </div>
         </div>
         <div id="folder-sharing" class="tab-pane">
-          <div class="form-group">
-            <label translate for="devices">Share With Devices</label>
+          <div class="form-group" ng-if="currentFolder.sharedDevices.length">
+            <label translate>Currently Shared With Devices</label>
             <p class="help-block">
-              <span translate>Select the devices to share this folder with.</span>&emsp;
-              <small><a href="#" ng-click="selectAllDevices()" translate>Select All</a>&emsp;
-                <a href="#" ng-click="deSelectAllDevices()" translate>Deselect All</a></small>
+              <span translate>Deselect devices to stop sharing this folder with.</span>&emsp;
+              <small><a href="#" ng-click="selectAllSharedDevices(true)" translate>Select All</a>&emsp;
+                <a href="#" ng-click="selectAllSharedDevices(false)" translate>Deselect All</a></small>
             </p>
             <div class="row">
-              <div class="col-md-4" ng-repeat="device in otherDevices()">
+              <div class="col-md-4" ng-repeat="device in currentFolder.sharedDevices">
+                <div class="checkbox">
+                  <label>
+                    <input type="checkbox" ng-model="currentFolder.selectedDevices[device.deviceID]" /> {{deviceName(device)}}
+                  </label>
+                </div>
+              </div>
+            </div>
+          </div>
+          <div class="form-group" ng-if="currentFolder.unrelatedDevices.length || otherDevices().length <= 0">
+            <label translate>Unshared Devices</label>
+            <p class="help-block" ng-if="otherDevices().length > 0">
+              <span translate>Select additional devices to share this folder with.</span>&emsp;
+              <small><a href="#" ng-click="selectAllUnrelatedDevices(true)" translate>Select All</a>&emsp;
+                <a href="#" ng-click="selectAllUnrelatedDevices(false)" translate>Deselect All</a></small>
+            </p>
+            <p class="help-block" ng-if="otherDevices().length <= 0">
+              <span translate>There are no devices to share this folder with.</span>
+            </p>
+            <div class="row">
+              <div class="col-md-4" ng-repeat="device in currentFolder.unrelatedDevices">
                 <div class="checkbox">
                   <label>
                     <input type="checkbox" ng-model="currentFolder.selectedDevices[device.deviceID]" /> {{deviceName(device)}}

gui/default/syncthing/settings/settingsModalView.html

@@ -94,7 +94,7 @@
                 <p class="help-block" ng-if="!upgradeInfo">
                   <span translate>Unavailable/Disabled by administrator or maintainer</span>
                 </p>
-                <p class="help-block" ng-if="version.isCandidate"">
+                <p class="help-block" ng-if="version.isCandidate && upgradeInfo"">
                   <span translate>Automatic upgrades are always enabled for candidate releases.</span>
                 </p>
               </div>

gui/light/assets/css/theme.css

@@ -0,0 +1,35 @@
+/*
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+*/
+
+.panel-progress {
+    background: #3498db;
+}
+
+.identicon rect {
+    fill: #333;
+}
+
+.panel-warning .identicon rect {
+    fill: #fff;
+}
+
+.li-column {
+    background-color: rgb(236, 240, 241);
+    border-radius: 3px;
+}
+
+.panel-heading:hover, .panel-heading:focus {
+    text-decoration: none;
+}
+
+.fancytree-ext-filter-hide tr.fancytree-submatch span.fancytree-title,
+.fancytree-ext-filter-hide span.fancytree-node.fancytree-submatch span.fancytree-title {
+    color: black !important;
+    font-weight: lighter !important;
+}

lib/api/api.go

@@ -9,10 +9,13 @@ package api
 import (
 	"bytes"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"net"
 	"net/http"
 	"net/url"
@@ -28,6 +31,10 @@ import (
 	"time"
 
 	metrics "github.com/rcrowley/go-metrics"
+	"github.com/thejerf/suture"
+	"github.com/vitrun/qart/qr"
+	"golang.org/x/crypto/bcrypt"
+
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/connections"
@@ -44,28 +51,30 @@ import (
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/upgrade"
 	"github.com/syncthing/syncthing/lib/ur"
-	"github.com/thejerf/suture"
-	"github.com/vitrun/qart/qr"
-	"golang.org/x/crypto/bcrypt"
+	"github.com/syncthing/syncthing/lib/util"
 )
 
 // matches a bcrypt hash and not too much else
 var bcryptExpr = regexp.MustCompile(`^\$2[aby]\$\d+\$.{50,}`)
 
 const (
-	DefaultEventMask    = events.AllEvents &^ events.LocalChangeDetected &^ events.RemoteChangeDetected
-	DiskEventMask       = events.LocalChangeDetected | events.RemoteChangeDetected
-	EventSubBufferSize  = 1000
-	defaultEventTimeout = time.Minute
+	DefaultEventMask      = events.AllEvents &^ events.LocalChangeDetected &^ events.RemoteChangeDetected
+	DiskEventMask         = events.LocalChangeDetected | events.RemoteChangeDetected
+	EventSubBufferSize    = 1000
+	defaultEventTimeout   = time.Minute
+	httpsCertLifetimeDays = 820
 )
 
 type service struct {
+	suture.Service
+
 	id                   protocol.DeviceID
 	cfg                  config.Wrapper
 	statics              *staticsServer
 	model                model.Model
 	eventSubs            map[events.EventType]events.BufferedSubscription
 	eventSubsMut         sync.Mutex
+	evLogger             events.Logger
 	discoverer           discover.CachingMux
 	connectionsService   connections.Service
 	fss                  model.FolderSummaryService
@@ -75,11 +84,11 @@ type service struct {
 	contr                Controller
 	noUpgrade            bool
 	tlsDefaultCommonName string
-	stop                 chan struct{} // signals intentional stop
 	configChanged        chan struct{} // signals intentional listener close due to config change
 	started              chan string   // signals startup complete by sending the listener address, for testing only
 	startedOnce          chan struct{} // the service has started successfully at least once
 	startupErr           error
+	listenerAddr         net.Addr
 
 	guiErrors logger.Recorder
 	systemLog logger.Recorder
@@ -101,8 +110,8 @@ type Service interface {
 	WaitForStart() error
 }
 
-func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonName string, m model.Model, defaultSub, diskSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService connections.Service, urService *ur.Service, fss model.FolderSummaryService, errors, systemLog logger.Recorder, cpu Rater, contr Controller, noUpgrade bool) Service {
-	return &service{
+func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonName string, m model.Model, defaultSub, diskSub events.BufferedSubscription, evLogger events.Logger, discoverer discover.CachingMux, connectionsService connections.Service, urService *ur.Service, fss model.FolderSummaryService, errors, systemLog logger.Recorder, cpu Rater, contr Controller, noUpgrade bool) Service {
+	s := &service{
 		id:      id,
 		cfg:     cfg,
 		statics: newStaticsServer(cfg.GUI().Theme, assetDir),
@@ -112,6 +121,7 @@ func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonNam
 			DiskEventMask:    diskSub,
 		},
 		eventSubsMut:         sync.NewMutex(),
+		evLogger:             evLogger,
 		discoverer:           discoverer,
 		connectionsService:   connectionsService,
 		fss:                  fss,
@@ -123,10 +133,11 @@ func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonNam
 		contr:                contr,
 		noUpgrade:            noUpgrade,
 		tlsDefaultCommonName: tlsDefaultCommonName,
-		stop:                 make(chan struct{}),
 		configChanged:        make(chan struct{}),
 		startedOnce:          make(chan struct{}),
 	}
+	s.Service = util.AsService(s.serve)
+	return s
 }
 
 func (s *service) WaitForStart() error {
@@ -138,6 +149,12 @@ func (s *service) getListener(guiCfg config.GUIConfiguration) (net.Listener, err
 	httpsCertFile := locations.Get(locations.HTTPSCertFile)
 	httpsKeyFile := locations.Get(locations.HTTPSKeyFile)
 	cert, err := tls.LoadX509KeyPair(httpsCertFile, httpsKeyFile)
+
+	// If the certificate has expired or will expire in the next month, fail
+	// it and generate a new one.
+	if err == nil {
+		err = checkExpiry(cert)
+	}
 	if err != nil {
 		l.Infoln("Loading HTTPS certificate:", err)
 		l.Infoln("Creating new HTTPS certificate")
@@ -150,7 +167,7 @@ func (s *service) getListener(guiCfg config.GUIConfiguration) (net.Listener, err
 			name = s.tlsDefaultCommonName
 		}
 
-		cert, err = tlsutil.NewCertificate(httpsCertFile, httpsKeyFile, name)
+		cert, err = tlsutil.NewCertificate(httpsCertFile, httpsKeyFile, name, httpsCertLifetimeDays)
 	}
 	if err != nil {
 		return nil, err
@@ -190,7 +207,7 @@ func sendJSON(w http.ResponseWriter, jsonObject interface{}) {
 	fmt.Fprintf(w, "%s\n", bs)
 }
 
-func (s *service) Serve() {
+func (s *service) serve(stop chan struct{}) {
 	listener, err := s.getListener(s.cfg.GUI())
 	if err != nil {
 		select {
@@ -215,6 +232,7 @@ func (s *service) Serve() {
 		return
 	}
 
+	s.listenerAddr = listener.Addr()
 	defer listener.Close()
 
 	s.cfg.Subscribe(s)
@@ -303,14 +321,14 @@ func (s *service) Serve() {
 
 	// Wrap everything in CSRF protection. The /rest prefix should be
 	// protected, other requests will grant cookies.
-	handler := csrfMiddleware(s.id.String()[:5], "/rest", guiCfg, mux)
+	var handler http.Handler = newCsrfManager(s.id.String()[:5], "/rest", guiCfg, mux, locations.Get(locations.CsrfTokens))
 
 	// Add our version and ID as a header to responses
 	handler = withDetailsMiddleware(s.id, handler)
 
 	// Wrap everything in basic auth, if user/password is set.
 	if guiCfg.IsAuthEnabled() {
-		handler = basicAuthAndSessionMiddleware("sessionid-"+s.id.String()[:5], guiCfg, s.cfg.LDAP(), handler)
+		handler = basicAuthAndSessionMiddleware("sessionid-"+s.id.String()[:5], guiCfg, s.cfg.LDAP(), handler, s.evLogger)
 	}
 
 	// Redirect to HTTPS if we are supposed to
@@ -333,6 +351,9 @@ func (s *service) Serve() {
 		// ReadTimeout must be longer than SyncthingController $scope.refresh
 		// interval to avoid HTTP keepalive/GUI refresh race.
 		ReadTimeout: 15 * time.Second,
+		// Prevent the HTTP server from logging stuff on its own. The things we
+		// care about we log ourselves from the handlers.
+		ErrorLog: log.New(ioutil.Discard, "", 0),
 	}
 
 	l.Infoln("GUI and API listening on", listener.Addr())
@@ -360,7 +381,7 @@ func (s *service) Serve() {
 	// Wait for stop, restart or error signals
 
 	select {
-	case <-s.stop:
+	case <-stop:
 		// Shutting down permanently
 		l.Debugln("shutting down (stop)")
 	case <-s.configChanged:
@@ -370,6 +391,7 @@ func (s *service) Serve() {
 		// Restart due to listen/serve failure
 		l.Warnln("GUI/API:", err, "(restarting)")
 	}
+	srv.Close()
 }
 
 // Complete implements suture.IsCompletable, which signifies to the supervisor
@@ -378,17 +400,11 @@ func (s *service) Complete() bool {
 	select {
 	case <-s.startedOnce:
 		return s.startupErr != nil
-	case <-s.stop:
-		return true
 	default:
 	}
 	return false
 }
 
-func (s *service) Stop() {
-	close(s.stop)
-}
-
 func (s *service) String() string {
 	return fmt.Sprintf("api.service@%p", s)
 }
@@ -908,6 +924,7 @@ func (s *service) getSystemStatus(w http.ResponseWriter, r *http.Request) {
 	res["uptime"] = s.urService.UptimeS()
 	res["startTime"] = ur.StartTime
 	res["guiAddressOverridden"] = s.cfg.GUI().IsOverridden()
+	res["guiAddressUsed"] = s.listenerAddr.String()
 
 	sendJSON(w, res)
 }
@@ -1212,7 +1229,7 @@ func (s *service) getEventSub(mask events.EventType) events.BufferedSubscription
 	s.eventSubsMut.Lock()
 	bufsub, ok := s.eventSubs[mask]
 	if !ok {
-		evsub := events.Default.Subscribe(mask)
+		evsub := s.evLogger.Subscribe(mask)
 		bufsub = events.NewBufferedSubscription(evsub, EventSubBufferSize)
 		s.eventSubs[mask] = bufsub
 	}
@@ -1565,10 +1582,10 @@ func (s *service) getHeapProf(w http.ResponseWriter, r *http.Request) {
 	pprof.WriteHeapProfile(w)
 }
 
-func toJsonFileInfoSlice(fs []db.FileInfoTruncated) []jsonDBFileInfo {
-	res := make([]jsonDBFileInfo, len(fs))
+func toJsonFileInfoSlice(fs []db.FileInfoTruncated) []jsonFileInfoTrunc {
+	res := make([]jsonFileInfoTrunc, len(fs))
 	for i, f := range fs {
-		res[i] = jsonDBFileInfo(f)
+		res[i] = jsonFileInfoTrunc(f)
 	}
 	return res
 }
@@ -1578,45 +1595,39 @@ func toJsonFileInfoSlice(fs []db.FileInfoTruncated) []jsonDBFileInfo {
 type jsonFileInfo protocol.FileInfo
 
 func (f jsonFileInfo) MarshalJSON() ([]byte, error) {
-	return json.Marshal(map[string]interface{}{
-		"name":          f.Name,
-		"type":          f.Type,
-		"size":          f.Size,
-		"permissions":   fmt.Sprintf("%#o", f.Permissions),
-		"deleted":       f.Deleted,
-		"invalid":       protocol.FileInfo(f).IsInvalid(),
-		"ignored":       protocol.FileInfo(f).IsIgnored(),
-		"mustRescan":    protocol.FileInfo(f).MustRescan(),
-		"noPermissions": f.NoPermissions,
-		"modified":      protocol.FileInfo(f).ModTime(),
-		"modifiedBy":    f.ModifiedBy.String(),
-		"sequence":      f.Sequence,
-		"numBlocks":     len(f.Blocks),
-		"version":       jsonVersionVector(f.Version),
-		"localFlags":    f.LocalFlags,
-	})
+	m := fileIntfJSONMap(protocol.FileInfo(f))
+	m["numBlocks"] = len(f.Blocks)
+	return json.Marshal(m)
 }
 
-type jsonDBFileInfo db.FileInfoTruncated
+type jsonFileInfoTrunc db.FileInfoTruncated
 
-func (f jsonDBFileInfo) MarshalJSON() ([]byte, error) {
-	return json.Marshal(map[string]interface{}{
-		"name":          f.Name,
-		"type":          f.Type.String(),
-		"size":          f.Size,
-		"permissions":   fmt.Sprintf("%#o", f.Permissions),
-		"deleted":       f.Deleted,
-		"invalid":       db.FileInfoTruncated(f).IsInvalid(),
-		"ignored":       db.FileInfoTruncated(f).IsIgnored(),
-		"mustRescan":    db.FileInfoTruncated(f).MustRescan(),
-		"noPermissions": f.NoPermissions,
-		"modified":      db.FileInfoTruncated(f).ModTime(),
-		"modifiedBy":    f.ModifiedBy.String(),
-		"sequence":      f.Sequence,
-		"numBlocks":     nil, // explicitly unknown
-		"version":       jsonVersionVector(f.Version),
-		"localFlags":    f.LocalFlags,
-	})
+func (f jsonFileInfoTrunc) MarshalJSON() ([]byte, error) {
+	m := fileIntfJSONMap(db.FileInfoTruncated(f))
+	m["numBlocks"] = nil // explicitly unknown
+	return json.Marshal(m)
+}
+
+func fileIntfJSONMap(f db.FileIntf) map[string]interface{} {
+	out := map[string]interface{}{
+		"name":          f.FileName(),
+		"type":          f.FileType().String(),
+		"size":          f.FileSize(),
+		"deleted":       f.IsDeleted(),
+		"invalid":       f.IsInvalid(),
+		"ignored":       f.IsIgnored(),
+		"mustRescan":    f.MustRescan(),
+		"noPermissions": !f.HasPermissionBits(),
+		"modified":      f.ModTime(),
+		"modifiedBy":    f.FileModifiedBy().String(),
+		"sequence":      f.SequenceNo(),
+		"version":       jsonVersionVector(f.FileVersion()),
+		"localFlags":    f.FileLocalFlags(),
+	}
+	if f.HasPermissionBits() {
+		out["permissions"] = fmt.Sprintf("%#o", f.FilePermissions())
+	}
+	return out
 }
 
 type jsonVersionVector protocol.Vector
@@ -1670,3 +1681,45 @@ func addressIsLocalhost(addr string) bool {
 		return ip.IsLoopback()
 	}
 }
+
+func checkExpiry(cert tls.Certificate) error {
+	leaf := cert.Leaf
+	if leaf == nil {
+		// Leaf can be nil or not, depending on how parsed the certificate
+		// was when we got it.
+		if len(cert.Certificate) < 1 {
+			// can't happen
+			return errors.New("no certificate in certificate")
+		}
+		var err error
+		leaf, err = x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return err
+		}
+	}
+
+	if leaf.Subject.String() != leaf.Issuer.String() ||
+		len(leaf.DNSNames) != 0 || len(leaf.IPAddresses) != 0 {
+		// The certificate is not self signed, or has DNS/IP attributes we don't
+		// add, so we leave it alone.
+		return nil
+	}
+
+	if leaf.NotAfter.Before(time.Now()) {
+		return errors.New("certificate has expired")
+	}
+	if leaf.NotAfter.Before(time.Now().Add(30 * 24 * time.Hour)) {
+		return errors.New("certificate will soon expire")
+	}
+
+	// On macOS, check for certificates issued on or after July 1st, 2019,
+	// with a longer validity time than 825 days.
+	cutoff := time.Date(2019, 7, 1, 0, 0, 0, 0, time.UTC)
+	if runtime.GOOS == "darwin" &&
+		leaf.NotBefore.After(cutoff) &&
+		leaf.NotAfter.Sub(leaf.NotBefore) > 825*24*time.Hour {
+		return errors.New("certificate incompatible with macOS 10.15 (Catalina)")
+	}
+
+	return nil
+}

lib/api/api_auth.go

@@ -28,14 +28,14 @@ var (
 	sessionsMut = sync.NewMutex()
 )
 
-func emitLoginAttempt(success bool, username string) {
-	events.Default.Log(events.LoginAttempt, map[string]interface{}{
+func emitLoginAttempt(success bool, username string, evLogger events.Logger) {
+	evLogger.Log(events.LoginAttempt, map[string]interface{}{
 		"success":  success,
 		"username": username,
 	})
 }
 
-func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfiguration, ldapCfg config.LDAPConfiguration, next http.Handler) http.Handler {
+func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfiguration, ldapCfg config.LDAPConfiguration, next http.Handler, evLogger events.Logger) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if guiCfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
 			next.ServeHTTP(w, r)
@@ -94,7 +94,7 @@ func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfigura
 		}
 
 		if !authOk {
-			emitLoginAttempt(false, username)
+			emitLoginAttempt(false, username, evLogger)
 			error()
 			return
 		}
@@ -109,7 +109,7 @@ func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfigura
 			MaxAge: 0,
 		})
 
-		emitLoginAttempt(true, username)
+		emitLoginAttempt(true, username, evLogger)
 		next.ServeHTTP(w, r)
 	})
 }

lib/api/api_auth_test.go

@@ -19,6 +19,8 @@ func init() {
 }
 
 func TestStaticAuthOK(t *testing.T) {
+	t.Parallel()
+
 	ok := authStatic("user", "pass", "user", string(passwordHashBytes))
 	if !ok {
 		t.Fatalf("should pass auth")
@@ -26,6 +28,8 @@ func TestStaticAuthOK(t *testing.T) {
 }
 
 func TestSimpleAuthUsernameFail(t *testing.T) {
+	t.Parallel()
+
 	ok := authStatic("userWRONG", "pass", "user", string(passwordHashBytes))
 	if ok {
 		t.Fatalf("should fail auth")
@@ -33,6 +37,8 @@ func TestSimpleAuthUsernameFail(t *testing.T) {
 }
 
 func TestStaticAuthPasswordFail(t *testing.T) {
+	t.Parallel()
+
 	ok := authStatic("user", "passWRONG", "user", string(passwordHashBytes))
 	if ok {
 		t.Fatalf("should fail auth")

lib/api/api_csrf.go

@@ -13,83 +13,103 @@ import (
 	"os"
 	"strings"
 
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
-// csrfTokens is a list of valid tokens. It is sorted so that the most
-// recently used token is first in the list. New tokens are added to the front
-// of the list (as it is the most recently used at that time). The list is
-// pruned to a maximum of maxCsrfTokens, throwing away the least recently used
-// tokens.
-var csrfTokens []string
-var csrfMut = sync.NewMutex()
-
 const maxCsrfTokens = 25
 
+type csrfManager struct {
+	// tokens is a list of valid tokens. It is sorted so that the most
+	// recently used token is first in the list. New tokens are added to the front
+	// of the list (as it is the most recently used at that time). The list is
+	// pruned to a maximum of maxCsrfTokens, throwing away the least recently used
+	// tokens.
+	tokens    []string
+	tokensMut sync.Mutex
+
+	unique          string
+	prefix          string
+	apiKeyValidator apiKeyValidator
+	next            http.Handler
+	saveLocation    string
+}
+
+type apiKeyValidator interface {
+	IsValidAPIKey(key string) bool
+}
+
 // Check for CSRF token on /rest/ URLs. If a correct one is not given, reject
 // the request with 403. For / and /index.html, set a new CSRF cookie if none
 // is currently set.
-func csrfMiddleware(unique string, prefix string, cfg config.GUIConfiguration, next http.Handler) http.Handler {
-	loadCsrfTokens()
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// Allow requests carrying a valid API key
-		if cfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
-			// Set the access-control-allow-origin header for CORS requests
-			// since a valid API key has been provided
-			w.Header().Add("Access-Control-Allow-Origin", "*")
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		if strings.HasPrefix(r.URL.Path, "/rest/debug") {
-			// Debugging functions are only available when explicitly
-			// enabled, and can be accessed without a CSRF token
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		// Allow requests for anything not under the protected path prefix,
-		// and set a CSRF cookie if there isn't already a valid one.
-		if !strings.HasPrefix(r.URL.Path, prefix) {
-			cookie, err := r.Cookie("CSRF-Token-" + unique)
-			if err != nil || !validCsrfToken(cookie.Value) {
-				l.Debugln("new CSRF cookie in response to request for", r.URL)
-				cookie = &http.Cookie{
-					Name:  "CSRF-Token-" + unique,
-					Value: newCsrfToken(),
-				}
-				http.SetCookie(w, cookie)
-			}
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		// Verify the CSRF token
-		token := r.Header.Get("X-CSRF-Token-" + unique)
-		if !validCsrfToken(token) {
-			http.Error(w, "CSRF Error", 403)
-			return
-		}
-
-		next.ServeHTTP(w, r)
-	})
+func newCsrfManager(unique string, prefix string, apiKeyValidator apiKeyValidator, next http.Handler, saveLocation string) *csrfManager {
+	m := &csrfManager{
+		tokensMut:       sync.NewMutex(),
+		unique:          unique,
+		prefix:          prefix,
+		apiKeyValidator: apiKeyValidator,
+		next:            next,
+		saveLocation:    saveLocation,
+	}
+	m.load()
+	return m
 }
 
-func validCsrfToken(token string) bool {
-	csrfMut.Lock()
-	defer csrfMut.Unlock()
-	for i, t := range csrfTokens {
+func (m *csrfManager) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Allow requests carrying a valid API key
+	if m.apiKeyValidator.IsValidAPIKey(r.Header.Get("X-API-Key")) {
+		// Set the access-control-allow-origin header for CORS requests
+		// since a valid API key has been provided
+		w.Header().Add("Access-Control-Allow-Origin", "*")
+		m.next.ServeHTTP(w, r)
+		return
+	}
+
+	if strings.HasPrefix(r.URL.Path, "/rest/debug") {
+		// Debugging functions are only available when explicitly
+		// enabled, and can be accessed without a CSRF token
+		m.next.ServeHTTP(w, r)
+		return
+	}
+
+	// Allow requests for anything not under the protected path prefix,
+	// and set a CSRF cookie if there isn't already a valid one.
+	if !strings.HasPrefix(r.URL.Path, m.prefix) {
+		cookie, err := r.Cookie("CSRF-Token-" + m.unique)
+		if err != nil || !m.validToken(cookie.Value) {
+			l.Debugln("new CSRF cookie in response to request for", r.URL)
+			cookie = &http.Cookie{
+				Name:  "CSRF-Token-" + m.unique,
+				Value: m.newToken(),
+			}
+			http.SetCookie(w, cookie)
+		}
+		m.next.ServeHTTP(w, r)
+		return
+	}
+
+	// Verify the CSRF token
+	token := r.Header.Get("X-CSRF-Token-" + m.unique)
+	if !m.validToken(token) {
+		http.Error(w, "CSRF Error", http.StatusForbidden)
+		return
+	}
+
+	m.next.ServeHTTP(w, r)
+}
+
+func (m *csrfManager) validToken(token string) bool {
+	m.tokensMut.Lock()
+	defer m.tokensMut.Unlock()
+	for i, t := range m.tokens {
 		if t == token {
 			if i > 0 {
 				// Move this token to the head of the list. Copy the tokens at
 				// the front one step to the right and then replace the token
 				// at the head.
-				copy(csrfTokens[1:], csrfTokens[:i+1])
-				csrfTokens[0] = token
+				copy(m.tokens[1:], m.tokens[:i+1])
+				m.tokens[0] = token
 			}
 			return true
 		}
@@ -97,40 +117,47 @@ func validCsrfToken(token string) bool {
 	return false
 }
 
-func newCsrfToken() string {
+func (m *csrfManager) newToken() string {
 	token := rand.String(32)
 
-	csrfMut.Lock()
-	csrfTokens = append([]string{token}, csrfTokens...)
-	if len(csrfTokens) > maxCsrfTokens {
-		csrfTokens = csrfTokens[:maxCsrfTokens]
+	m.tokensMut.Lock()
+	m.tokens = append([]string{token}, m.tokens...)
+	if len(m.tokens) > maxCsrfTokens {
+		m.tokens = m.tokens[:maxCsrfTokens]
 	}
-	defer csrfMut.Unlock()
+	defer m.tokensMut.Unlock()
 
-	saveCsrfTokens()
+	m.save()
 
 	return token
 }
 
-func saveCsrfTokens() {
+func (m *csrfManager) save() {
 	// We're ignoring errors in here. It's not super critical and there's
 	// nothing relevant we can do about them anyway...
 
-	name := locations.Get(locations.CsrfTokens)
-	f, err := osutil.CreateAtomic(name)
+	if m.saveLocation == "" {
+		return
+	}
+
+	f, err := osutil.CreateAtomic(m.saveLocation)
 	if err != nil {
 		return
 	}
 
-	for _, t := range csrfTokens {
+	for _, t := range m.tokens {
 		fmt.Fprintln(f, t)
 	}
 
 	f.Close()
 }
 
-func loadCsrfTokens() {
-	f, err := os.Open(locations.Get(locations.CsrfTokens))
+func (m *csrfManager) load() {
+	if m.saveLocation == "" {
+		return
+	}
+
+	f, err := os.Open(m.saveLocation)
 	if err != nil {
 		return
 	}
@@ -138,6 +165,6 @@ func loadCsrfTokens() {
 
 	s := bufio.NewScanner(f)
 	for s.Scan() {
-		csrfTokens = append(csrfTokens, s.Text())
+		m.tokens = append(m.tokens, s.Text())
 	}
 }

lib/api/api_statics.go

@@ -23,21 +23,25 @@ import (
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
+const themePrefix = "theme-assets/"
+
 type staticsServer struct {
 	assetDir        string
 	assets          map[string][]byte
 	availableThemes []string
 
-	mut   sync.RWMutex
-	theme string
+	mut             sync.RWMutex
+	theme           string
+	lastThemeChange time.Time
 }
 
 func newStaticsServer(theme, assetDir string) *staticsServer {
 	s := &staticsServer{
-		assetDir: assetDir,
-		assets:   auto.Assets(),
-		mut:      sync.NewRWMutex(),
-		theme:    theme,
+		assetDir:        assetDir,
+		assets:          auto.Assets(),
+		mut:             sync.NewRWMutex(),
+		theme:           theme,
+		lastThemeChange: time.Now().UTC(),
 	}
 
 	seen := make(map[string]struct{})
@@ -86,8 +90,23 @@ func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) {
 
 	s.mut.RLock()
 	theme := s.theme
+	modificationTime := s.lastThemeChange
 	s.mut.RUnlock()
 
+	// If path starts with special prefix, get theme and file from path
+	if strings.HasPrefix(file, themePrefix) {
+		path := file[len(themePrefix):]
+		i := strings.IndexRune(path, '/')
+
+		if i == -1 {
+			http.NotFound(w, r)
+			return
+		}
+
+		theme = path[:i]
+		file = path[i+1:]
+	}
+
 	// Check for an override for the current theme.
 	if s.assetDir != "" {
 		p := filepath.Join(s.assetDir, theme, filepath.FromSlash(file))
@@ -125,14 +144,12 @@ func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	etag := fmt.Sprintf("%d", auto.Generated)
-	modified := time.Unix(auto.Generated, 0).UTC()
-
-	w.Header().Set("Last-Modified", modified.Format(http.TimeFormat))
+	etag := fmt.Sprintf("%d", modificationTime.Unix())
+	w.Header().Set("Last-Modified", modificationTime.Format(http.TimeFormat))
 	w.Header().Set("Etag", etag)
 
 	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil {
-		if modified.Equal(t) || modified.Before(t) {
+		if modificationTime.Equal(t) || modificationTime.Before(t) {
 			w.WriteHeader(http.StatusNotModified)
 			return
 		}
@@ -199,6 +216,7 @@ func (s *staticsServer) mimeTypeForFile(file string) string {
 func (s *staticsServer) setTheme(theme string) {
 	s.mut.Lock()
 	s.theme = theme
+	s.lastThemeChange = time.Now().UTC()
 	s.mut.Unlock()
 }
 

lib/api/api_test.go

@@ -18,6 +18,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"testing"
@@ -31,6 +32,7 @@ import (
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/ur"
 	"github.com/thejerf/suture"
 )
@@ -52,7 +54,7 @@ func TestMain(m *testing.M) {
 }
 
 func TestCSRFToken(t *testing.T) {
-	defer os.Remove(token)
+	t.Parallel()
 
 	max := 250
 	int := 5
@@ -61,11 +63,13 @@ func TestCSRFToken(t *testing.T) {
 		int = 2
 	}
 
-	t1 := newCsrfToken()
-	t2 := newCsrfToken()
+	m := newCsrfManager("unique", "prefix", config.GUIConfiguration{}, nil, "")
 
-	t3 := newCsrfToken()
-	if !validCsrfToken(t3) {
+	t1 := m.newToken()
+	t2 := m.newToken()
+
+	t3 := m.newToken()
+	if !m.validToken(t3) {
 		t.Fatal("t3 should be valid")
 	}
 
@@ -73,36 +77,38 @@ func TestCSRFToken(t *testing.T) {
 		if i%int == 0 {
 			// t1 and t2 should remain valid by virtue of us checking them now
 			// and then.
-			if !validCsrfToken(t1) {
+			if !m.validToken(t1) {
 				t.Fatal("t1 should be valid at iteration", i)
 			}
-			if !validCsrfToken(t2) {
+			if !m.validToken(t2) {
 				t.Fatal("t2 should be valid at iteration", i)
 			}
 		}
 
 		// The newly generated token is always valid
-		t4 := newCsrfToken()
-		if !validCsrfToken(t4) {
+		t4 := m.newToken()
+		if !m.validToken(t4) {
 			t.Fatal("t4 should be valid at iteration", i)
 		}
 	}
 
-	if validCsrfToken(t3) {
+	if m.validToken(t3) {
 		t.Fatal("t3 should have expired by now")
 	}
 }
 
 func TestStopAfterBrokenConfig(t *testing.T) {
+	t.Parallel()
+
 	cfg := config.Configuration{
 		GUI: config.GUIConfiguration{
 			RawAddress: "127.0.0.1:0",
 			RawUseTLS:  false,
 		},
 	}
-	w := config.Wrap("/dev/null", cfg)
+	w := config.Wrap("/dev/null", cfg, events.NoopLogger)
 
-	srv := New(protocol.LocalDeviceID, w, "", "syncthing", nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, false).(*service)
+	srv := New(protocol.LocalDeviceID, w, "", "syncthing", nil, nil, nil, events.NoopLogger, nil, nil, nil, nil, nil, nil, nil, nil, false).(*service)
 	defer os.Remove(token)
 	srv.started = make(chan string)
 
@@ -134,6 +140,8 @@ func TestStopAfterBrokenConfig(t *testing.T) {
 }
 
 func TestAssetsDir(t *testing.T) {
+	t.Parallel()
+
 	// For any given request to $FILE, we should return the first found of
 	//  - assetsdir/$THEME/$FILE
 	//  - compiled in asset $THEME/$FILE
@@ -208,6 +216,8 @@ func expectURLToContain(t *testing.T, url, exp string) {
 }
 
 func TestDirNames(t *testing.T) {
+	t.Parallel()
+
 	names := dirNames("testdata")
 	expected := []string{"config", "default", "foo", "testfolder"}
 	if diff, equal := messagediff.PrettyDiff(expected, names); !equal {
@@ -224,6 +234,8 @@ type httpTestCase struct {
 }
 
 func TestAPIServiceRequests(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -434,6 +446,8 @@ func testHTTPRequest(t *testing.T, baseURL string, tc httpTestCase, apikey strin
 }
 
 func TestHTTPLogin(t *testing.T) {
+	t.Parallel()
+
 	cfg := new(mockedConfig)
 	cfg.gui.User = "üser"
 	cfg.gui.Password = "$2a$10$IdIZTxTg/dCNuNEGlmLynOjqg4B1FvDKuIV5e0BB3pnWVHNb8.GSq" // bcrypt of "räksmörgås" in UTF-8
@@ -512,8 +526,8 @@ func startHTTP(cfg *mockedConfig) (string, error) {
 
 	// Instantiate the API service
 	urService := ur.New(cfg, m, connections, false)
-	summaryService := model.NewFolderSummaryService(cfg, m, protocol.LocalDeviceID)
-	svc := New(protocol.LocalDeviceID, cfg, assetDir, "syncthing", m, eventSub, diskEventSub, discoverer, connections, urService, summaryService, errorLog, systemLog, cpu, nil, false).(*service)
+	summaryService := model.NewFolderSummaryService(cfg, m, protocol.LocalDeviceID, events.NoopLogger)
+	svc := New(protocol.LocalDeviceID, cfg, assetDir, "syncthing", m, eventSub, diskEventSub, events.NoopLogger, discoverer, connections, urService, summaryService, errorLog, systemLog, cpu, nil, false).(*service)
 	defer os.Remove(token)
 	svc.started = addrChan
 
@@ -541,6 +555,8 @@ func startHTTP(cfg *mockedConfig) (string, error) {
 }
 
 func TestCSRFRequired(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -550,7 +566,7 @@ func TestCSRFRequired(t *testing.T) {
 	}
 
 	cli := &http.Client{
-		Timeout: time.Second,
+		Timeout: time.Minute,
 	}
 
 	// Getting the base URL (i.e. "/") should succeed.
@@ -614,6 +630,8 @@ func TestCSRFRequired(t *testing.T) {
 }
 
 func TestRandomString(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -663,10 +681,15 @@ func TestRandomString(t *testing.T) {
 }
 
 func TestConfigPostOK(t *testing.T) {
+	t.Parallel()
+
 	cfg := bytes.NewBuffer([]byte(`{
 		"version": 15,
 		"folders": [
-			{"id": "foo"}
+			{
+				"id": "foo",
+				"path": "TestConfigPostOK"
+			}
 		]
 	}`))
 
@@ -677,9 +700,12 @@ func TestConfigPostOK(t *testing.T) {
 	if resp.StatusCode != http.StatusOK {
 		t.Error("Expected 200 OK, not", resp.Status)
 	}
+	os.RemoveAll("TestConfigPostOK")
 }
 
 func TestConfigPostDupFolder(t *testing.T) {
+	t.Parallel()
+
 	cfg := bytes.NewBuffer([]byte(`{
 		"version": 15,
 		"folders": [
@@ -715,6 +741,8 @@ func testConfigPost(data io.Reader) (*http.Response, error) {
 }
 
 func TestHostCheck(t *testing.T) {
+	t.Parallel()
+
 	// An API service bound to localhost should reject non-localhost host Headers
 
 	cfg := new(mockedConfig)
@@ -822,6 +850,11 @@ func TestHostCheck(t *testing.T) {
 
 	// This should all work over IPv6 as well
 
+	if runningInContainer() {
+		// Working IPv6 in Docker can't be taken for granted.
+		return
+	}
+
 	cfg = new(mockedConfig)
 	cfg.gui.RawAddress = "[::1]:0"
 	baseURL, err = startHTTP(cfg)
@@ -868,6 +901,8 @@ func TestHostCheck(t *testing.T) {
 }
 
 func TestAddressIsLocalhost(t *testing.T) {
+	t.Parallel()
+
 	testcases := []struct {
 		address string
 		result  bool
@@ -911,6 +946,8 @@ func TestAddressIsLocalhost(t *testing.T) {
 }
 
 func TestAccessControlAllowOriginHeader(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -939,6 +976,8 @@ func TestAccessControlAllowOriginHeader(t *testing.T) {
 }
 
 func TestOptionsRequest(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -972,10 +1011,12 @@ func TestOptionsRequest(t *testing.T) {
 }
 
 func TestEventMasks(t *testing.T) {
+	t.Parallel()
+
 	cfg := new(mockedConfig)
 	defSub := new(mockedEventSub)
 	diskSub := new(mockedEventSub)
-	svc := New(protocol.LocalDeviceID, cfg, "", "syncthing", nil, defSub, diskSub, nil, nil, nil, nil, nil, nil, nil, nil, false).(*service)
+	svc := New(protocol.LocalDeviceID, cfg, "", "syncthing", nil, defSub, diskSub, events.NoopLogger, nil, nil, nil, nil, nil, nil, nil, nil, false).(*service)
 	defer os.Remove(token)
 
 	if mask := svc.getEventMask(""); mask != DefaultEventMask {
@@ -1004,6 +1045,8 @@ func TestEventMasks(t *testing.T) {
 }
 
 func TestBrowse(t *testing.T) {
+	t.Parallel()
+
 	pathSep := string(os.PathSeparator)
 
 	tmpDir, err := ioutil.TempDir("", "syncthing")
@@ -1056,6 +1099,8 @@ func TestBrowse(t *testing.T) {
 }
 
 func TestPrefixMatch(t *testing.T) {
+	t.Parallel()
+
 	cases := []struct {
 		s        string
 		prefix   string
@@ -1075,6 +1120,44 @@ func TestPrefixMatch(t *testing.T) {
 	}
 }
 
+func TestCheckExpiry(t *testing.T) {
+	dir, err := ioutil.TempDir("", "syncthing-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	// Self signed certificates expiring in less than a month are errored so we
+	// can regenerate in time.
+	crt, err := tlsutil.NewCertificate(filepath.Join(dir, "crt"), filepath.Join(dir, "key"), "foo.example.com", 29)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := checkExpiry(crt); err == nil {
+		t.Error("expected expiry error")
+	}
+
+	// Certificates with at least 31 days of life left are fine.
+	crt, err = tlsutil.NewCertificate(filepath.Join(dir, "crt"), filepath.Join(dir, "key"), "foo.example.com", 31)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := checkExpiry(crt); err != nil {
+		t.Error("expected no error:", err)
+	}
+
+	if runtime.GOOS == "darwin" {
+		// Certificates with too long an expiry time are not allowed on macOS
+		crt, err = tlsutil.NewCertificate(filepath.Join(dir, "crt"), filepath.Join(dir, "key"), "foo.example.com", 1000)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := checkExpiry(crt); err == nil {
+			t.Error("expected expiry error")
+		}
+	}
+}
+
 func equalStrings(a, b []string) bool {
 	if len(a) != len(b) {
 		return false
@@ -1086,3 +1169,24 @@ func equalStrings(a, b []string) bool {
 	}
 	return true
 }
+
+// runningInContainer returns true if we are inside Docker or LXC. It might
+// be prone to false negatives if things change in the future, but likely
+// not false positives.
+func runningInContainer() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+
+	bs, err := ioutil.ReadFile("/proc/1/cgroup")
+	if err != nil {
+		return false
+	}
+	if bytes.Contains(bs, []byte("/docker/")) {
+		return true
+	}
+	if bytes.Contains(bs, []byte("/lxc/")) {
+		return true
+	}
+	return false
+}

lib/api/debug.go

@@ -7,9 +7,6 @@
 package api
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
@@ -24,5 +21,7 @@ func shouldDebugHTTP() bool {
 func init() {
 	// The debug facility was originally named "http", changed in:
 	// https://github.com/syncthing/syncthing/pull/5548
-	l.SetDebug("api", strings.Contains(os.Getenv("STTRACE"), "api") || strings.Contains(os.Getenv("STTRACE"), "http") || os.Getenv("STTRACE") == "all")
+	if l.IsTraced("http") {
+		l.SetDebug("api", true)
+	}
 }

lib/api/mocked_connections_test.go

@@ -27,3 +27,7 @@ func (m *mockedConnections) NATType() string {
 func (m *mockedConnections) Serve() {}
 
 func (m *mockedConnections) Stop() {}
+
+func (m *mockedConnections) ExternalAddresses() []string { return nil }
+
+func (m *mockedConnections) AllAddresses() []string { return nil }

lib/beacon/beacon.go

@@ -7,10 +7,13 @@
 package beacon
 
 import (
+	"fmt"
 	"net"
-	stdsync "sync"
+	"time"
 
 	"github.com/thejerf/suture"
+
+	"github.com/syncthing/syncthing/lib/util"
 )
 
 type recv struct {
@@ -20,25 +23,93 @@ type recv struct {
 
 type Interface interface {
 	suture.Service
+	fmt.Stringer
 	Send(data []byte)
 	Recv() ([]byte, net.Addr)
 	Error() error
 }
 
-type errorHolder struct {
-	err error
-	mut stdsync.Mutex // uses stdlib sync as I want this to be trivially embeddable, and there is no risk of blocking
+type cast struct {
+	*suture.Supervisor
+	name    string
+	reader  util.ServiceWithError
+	writer  util.ServiceWithError
+	outbox  chan recv
+	inbox   chan []byte
+	stopped chan struct{}
 }
 
-func (e *errorHolder) setError(err error) {
-	e.mut.Lock()
-	e.err = err
-	e.mut.Unlock()
+// newCast creates a base object for multi- or broadcasting. Afterwards the
+// caller needs to set reader and writer with the addReader and addWriter
+// methods to get a functional implementation of Interface.
+func newCast(name string) *cast {
+	return &cast{
+		Supervisor: suture.New(name, suture.Spec{
+			// Don't retry too frenetically: an error to open a socket or
+			// whatever is usually something that is either permanent or takes
+			// a while to get solved...
+			FailureThreshold: 2,
+			FailureBackoff:   60 * time.Second,
+			// Only log restarts in debug mode.
+			Log: func(line string) {
+				l.Debugln(line)
+			},
+			PassThroughPanics: true,
+		}),
+		name:    name,
+		inbox:   make(chan []byte),
+		outbox:  make(chan recv, 16),
+		stopped: make(chan struct{}),
+	}
 }
 
-func (e *errorHolder) Error() error {
-	e.mut.Lock()
-	err := e.err
-	e.mut.Unlock()
-	return err
+func (c *cast) addReader(svc func(chan struct{}) error) {
+	c.reader = c.createService(svc, "reader")
+	c.Add(c.reader)
+}
+
+func (c *cast) addWriter(svc func(stop chan struct{}) error) {
+	c.writer = c.createService(svc, "writer")
+	c.Add(c.writer)
+}
+
+func (c *cast) createService(svc func(chan struct{}) error, suffix string) util.ServiceWithError {
+	return util.AsServiceWithError(func(stop chan struct{}) error {
+		l.Debugln("Starting", c.name, suffix)
+		err := svc(stop)
+		l.Debugf("Stopped %v %v: %v", c.name, suffix, err)
+		return err
+	})
+}
+
+func (c *cast) Stop() {
+	c.Supervisor.Stop()
+	close(c.stopped)
+}
+
+func (c *cast) String() string {
+	return fmt.Sprintf("%s@%p", c.name, c)
+}
+
+func (c *cast) Send(data []byte) {
+	select {
+	case c.inbox <- data:
+	case <-c.stopped:
+	}
+}
+
+func (c *cast) Recv() ([]byte, net.Addr) {
+	select {
+	case recv := <-c.outbox:
+		return recv.data, recv.src
+	case <-c.stopped:
+	}
+	return nil, nil
+}
+
+func (c *cast) Error() error {
+	if err := c.reader.Error(); err != nil {
+		return err
+	}
+	return c.writer.Error()
 }

lib/beacon/broadcast.go

@@ -7,104 +7,49 @@
 package beacon
 
 import (
-	"fmt"
 	"net"
 	"time"
-
-	"github.com/syncthing/syncthing/lib/sync"
-	"github.com/thejerf/suture"
 )
 
-type Broadcast struct {
-	*suture.Supervisor
-	port   int
-	inbox  chan []byte
-	outbox chan recv
-	br     *broadcastReader
-	bw     *broadcastWriter
+func NewBroadcast(port int) Interface {
+	c := newCast("broadcastBeacon")
+	c.addReader(func(stop chan struct{}) error {
+		return readBroadcasts(c.outbox, port, stop)
+	})
+	c.addWriter(func(stop chan struct{}) error {
+		return writeBroadcasts(c.inbox, port, stop)
+	})
+	return c
 }
 
-func NewBroadcast(port int) *Broadcast {
-	b := &Broadcast{
-		Supervisor: suture.New("broadcastBeacon", suture.Spec{
-			// Don't retry too frenetically: an error to open a socket or
-			// whatever is usually something that is either permanent or takes
-			// a while to get solved...
-			FailureThreshold: 2,
-			FailureBackoff:   60 * time.Second,
-			// Only log restarts in debug mode.
-			Log: func(line string) {
-				l.Debugln(line)
-			},
-			PassThroughPanics: true,
-		}),
-		port:   port,
-		inbox:  make(chan []byte),
-		outbox: make(chan recv, 16),
-	}
-
-	b.br = &broadcastReader{
-		port:    port,
-		outbox:  b.outbox,
-		connMut: sync.NewMutex(),
-	}
-	b.Add(b.br)
-	b.bw = &broadcastWriter{
-		port:    port,
-		inbox:   b.inbox,
-		connMut: sync.NewMutex(),
-	}
-	b.Add(b.bw)
-
-	return b
-}
-
-func (b *Broadcast) Send(data []byte) {
-	b.inbox <- data
-}
-
-func (b *Broadcast) Recv() ([]byte, net.Addr) {
-	recv := <-b.outbox
-	return recv.data, recv.src
-}
-
-func (b *Broadcast) Error() error {
-	if err := b.br.Error(); err != nil {
-		return err
-	}
-	return b.bw.Error()
-}
-
-type broadcastWriter struct {
-	port    int
-	inbox   chan []byte
-	conn    *net.UDPConn
-	connMut sync.Mutex
-	errorHolder
-}
-
-func (w *broadcastWriter) Serve() {
-	l.Debugln(w, "starting")
-	defer l.Debugln(w, "stopping")
-
+func writeBroadcasts(inbox <-chan []byte, port int, stop chan struct{}) error {
 	conn, err := net.ListenUDP("udp4", nil)
 	if err != nil {
 		l.Debugln(err)
-		w.setError(err)
-		return
+		return err
 	}
-	defer conn.Close()
+	done := make(chan struct{})
+	defer close(done)
+	go func() {
+		select {
+		case <-stop:
+		case <-done:
+		}
+		conn.Close()
+	}()
 
-	w.connMut.Lock()
-	w.conn = conn
-	w.connMut.Unlock()
+	for {
+		var bs []byte
+		select {
+		case bs = <-inbox:
+		case <-stop:
+			return nil
+		}
 
-	for bs := range w.inbox {
 		addrs, err := net.InterfaceAddrs()
 		if err != nil {
 			l.Debugln(err)
-			w.setError(err)
-			continue
+			return err
 		}
 
 		var dsts []net.IP
@@ -124,24 +69,22 @@ func (w *broadcastWriter) Serve() {
 
 		success := 0
 		for _, ip := range dsts {
-			dst := &net.UDPAddr{IP: ip, Port: w.port}
+			dst := &net.UDPAddr{IP: ip, Port: port}
 
 			conn.SetWriteDeadline(time.Now().Add(time.Second))
-			_, err := conn.WriteTo(bs, dst)
+			_, err = conn.WriteTo(bs, dst)
 			conn.SetWriteDeadline(time.Time{})
 
-			if err, ok := err.(net.Error); ok && err.Timeout() {
+			if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
 				// Write timeouts should not happen. We treat it as a fatal
 				// error on the socket.
 				l.Debugln(err)
-				w.setError(err)
-				return
+				return err
 			}
 
 			if err != nil {
 				// Some other error that we don't expect. Debug and continue.
 				l.Debugln(err)
-				w.setError(err)
 				continue
 			}
 
@@ -149,82 +92,49 @@ func (w *broadcastWriter) Serve() {
 			success++
 		}
 
-		if success > 0 {
-			w.setError(nil)
+		if success == 0 {
+			l.Debugln("couldn't send any braodcasts")
+			return err
 		}
 	}
 }
 
-func (w *broadcastWriter) Stop() {
-	w.connMut.Lock()
-	if w.conn != nil {
-		w.conn.Close()
-	}
-	w.connMut.Unlock()
-}
-
-func (w *broadcastWriter) String() string {
-	return fmt.Sprintf("broadcastWriter@%p", w)
-}
-
-type broadcastReader struct {
-	port    int
-	outbox  chan recv
-	conn    *net.UDPConn
-	connMut sync.Mutex
-	errorHolder
-}
-
-func (r *broadcastReader) Serve() {
-	l.Debugln(r, "starting")
-	defer l.Debugln(r, "stopping")
-
-	conn, err := net.ListenUDP("udp4", &net.UDPAddr{Port: r.port})
+func readBroadcasts(outbox chan<- recv, port int, stop chan struct{}) error {
+	conn, err := net.ListenUDP("udp4", &net.UDPAddr{Port: port})
 	if err != nil {
 		l.Debugln(err)
-		r.setError(err)
-		return
+		return err
 	}
-	defer conn.Close()
-
-	r.connMut.Lock()
-	r.conn = conn
-	r.connMut.Unlock()
+	done := make(chan struct{})
+	defer close(done)
+	go func() {
+		select {
+		case <-stop:
+		case <-done:
+		}
+		conn.Close()
+	}()
 
 	bs := make([]byte, 65536)
 	for {
 		n, addr, err := conn.ReadFrom(bs)
 		if err != nil {
 			l.Debugln(err)
-			r.setError(err)
-			return
+			return err
 		}
 
-		r.setError(nil)
-
 		l.Debugf("recv %d bytes from %s", n, addr)
 
 		c := make([]byte, n)
 		copy(c, bs)
 		select {
-		case r.outbox <- recv{c, addr}:
+		case outbox <- recv{c, addr}:
+		case <-stop:
+			return nil
 		default:
 			l.Debugln("dropping message")
 		}
 	}
-
-}
-
-func (r *broadcastReader) Stop() {
-	r.connMut.Lock()
-	if r.conn != nil {
-		r.conn.Close()
-	}
-	r.connMut.Unlock()
-}
-
-func (r *broadcastReader) String() string {
-	return fmt.Sprintf("broadcastReader@%p", r)
 }
 
 func bcast(ip *net.IPNet) *net.IPNet {

lib/beacon/debug.go

@@ -7,16 +7,9 @@
 package beacon
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
 	l = logger.DefaultLogger.NewFacility("beacon", "Multicast and broadcast discovery")
 )
-
-func init() {
-	l.SetDebug("beacon", strings.Contains(os.Getenv("STTRACE"), "beacon") || os.Getenv("STTRACE") == "all")
-}

lib/beacon/multicast.go

@@ -8,97 +8,44 @@ package beacon
 
 import (
 	"errors"
-	"fmt"
 	"net"
 	"time"
 
-	"github.com/thejerf/suture"
 	"golang.org/x/net/ipv6"
 )
 
-type Multicast struct {
-	*suture.Supervisor
-	inbox  chan []byte
-	outbox chan recv
-	mr     *multicastReader
-	mw     *multicastWriter
+func NewMulticast(addr string) Interface {
+	c := newCast("multicastBeacon")
+	c.addReader(func(stop chan struct{}) error {
+		return readMulticasts(c.outbox, addr, stop)
+	})
+	c.addWriter(func(stop chan struct{}) error {
+		return writeMulticasts(c.inbox, addr, stop)
+	})
+	return c
 }
 
-func NewMulticast(addr string) *Multicast {
-	m := &Multicast{
-		Supervisor: suture.New("multicastBeacon", suture.Spec{
-			// Don't retry too frenetically: an error to open a socket or
-			// whatever is usually something that is either permanent or takes
-			// a while to get solved...
-			FailureThreshold: 2,
-			FailureBackoff:   60 * time.Second,
-			// Only log restarts in debug mode.
-			Log: func(line string) {
-				l.Debugln(line)
-			},
-			PassThroughPanics: true,
-		}),
-		inbox:  make(chan []byte),
-		outbox: make(chan recv, 16),
-	}
-
-	m.mr = &multicastReader{
-		addr:   addr,
-		outbox: m.outbox,
-		stop:   make(chan struct{}),
-	}
-	m.Add(m.mr)
-
-	m.mw = &multicastWriter{
-		addr:  addr,
-		inbox: m.inbox,
-		stop:  make(chan struct{}),
-	}
-	m.Add(m.mw)
-
-	return m
-}
-
-func (m *Multicast) Send(data []byte) {
-	m.inbox <- data
-}
-
-func (m *Multicast) Recv() ([]byte, net.Addr) {
-	recv := <-m.outbox
-	return recv.data, recv.src
-}
-
-func (m *Multicast) Error() error {
-	if err := m.mr.Error(); err != nil {
-		return err
-	}
-	return m.mw.Error()
-}
-
-type multicastWriter struct {
-	addr  string
-	inbox <-chan []byte
-	errorHolder
-	stop chan struct{}
-}
-
-func (w *multicastWriter) Serve() {
-	l.Debugln(w, "starting")
-	defer l.Debugln(w, "stopping")
-
-	gaddr, err := net.ResolveUDPAddr("udp6", w.addr)
+func writeMulticasts(inbox <-chan []byte, addr string, stop chan struct{}) error {
+	gaddr, err := net.ResolveUDPAddr("udp6", addr)
 	if err != nil {
 		l.Debugln(err)
-		w.setError(err)
-		return
+		return err
 	}
 
 	conn, err := net.ListenPacket("udp6", ":0")
 	if err != nil {
 		l.Debugln(err)
-		w.setError(err)
-		return
+		return err
 	}
+	done := make(chan struct{})
+	defer close(done)
+	go func() {
+		select {
+		case <-stop:
+		case <-done:
+		}
+		conn.Close()
+	}()
 
 	pconn := ipv6.NewPacketConn(conn)
 
@@ -106,12 +53,18 @@ func (w *multicastWriter) Serve() {
 		HopLimit: 1,
 	}
 
-	for bs := range w.inbox {
+	for {
+		var bs []byte
+		select {
+		case bs = <-inbox:
+		case <-stop:
+			return nil
+		}
+
 		intfs, err := net.Interfaces()
 		if err != nil {
 			l.Debugln(err)
-			w.setError(err)
-			return
+			return err
 		}
 
 		success := 0
@@ -123,62 +76,52 @@ func (w *multicastWriter) Serve() {
 
 			if err != nil {
 				l.Debugln(err, "on write to", gaddr, intf.Name)
-				w.setError(err)
 				continue
 			}
 
 			l.Debugf("sent %d bytes to %v on %s", len(bs), gaddr, intf.Name)
 
 			success++
+
+			select {
+			case <-stop:
+				return nil
+			default:
+			}
 		}
 
-		if success > 0 {
-			w.setError(nil)
-		} else {
-			l.Debugln(err)
-			w.setError(err)
+		if success == 0 {
+			return err
 		}
 	}
 }
 
-func (w *multicastWriter) Stop() {
-	close(w.stop)
-}
-
-func (w *multicastWriter) String() string {
-	return fmt.Sprintf("multicastWriter@%p", w)
-}
-
-type multicastReader struct {
-	addr   string
-	outbox chan<- recv
-	errorHolder
-	stop chan struct{}
-}
-
-func (r *multicastReader) Serve() {
-	l.Debugln(r, "starting")
-	defer l.Debugln(r, "stopping")
-
-	gaddr, err := net.ResolveUDPAddr("udp6", r.addr)
+func readMulticasts(outbox chan<- recv, addr string, stop chan struct{}) error {
+	gaddr, err := net.ResolveUDPAddr("udp6", addr)
 	if err != nil {
 		l.Debugln(err)
-		r.setError(err)
-		return
+		return err
 	}
 
-	conn, err := net.ListenPacket("udp6", r.addr)
+	conn, err := net.ListenPacket("udp6", addr)
 	if err != nil {
 		l.Debugln(err)
-		r.setError(err)
-		return
+		return err
 	}
+	done := make(chan struct{})
+	defer close(done)
+	go func() {
+		select {
+		case <-stop:
+		case <-done:
+		}
+		conn.Close()
+	}()
 
 	intfs, err := net.Interfaces()
 	if err != nil {
 		l.Debugln(err)
-		r.setError(err)
-		return
+		return err
 	}
 
 	pconn := ipv6.NewPacketConn(conn)
@@ -195,34 +138,29 @@ func (r *multicastReader) Serve() {
 
 	if joined == 0 {
 		l.Debugln("no multicast interfaces available")
-		r.setError(errors.New("no multicast interfaces available"))
-		return
+		return errors.New("no multicast interfaces available")
 	}
 
 	bs := make([]byte, 65536)
 	for {
+		select {
+		case <-stop:
+			return nil
+		default:
+		}
 		n, _, addr, err := pconn.ReadFrom(bs)
 		if err != nil {
 			l.Debugln(err)
-			r.setError(err)
-			continue
+			return err
 		}
 		l.Debugf("recv %d bytes from %s", n, addr)
 
 		c := make([]byte, n)
 		copy(c, bs)
 		select {
-		case r.outbox <- recv{c, addr}:
+		case outbox <- recv{c, addr}:
 		default:
 			l.Debugln("dropping message")
 		}
 	}
 }
-
-func (r *multicastReader) Stop() {
-	close(r.stop)
-}
-
-func (r *multicastReader) String() string {
-	return fmt.Sprintf("multicastReader@%p", r)
-}

lib/build/build.go

@@ -18,10 +18,11 @@ import (
 
 var (
 	// Injected by build script
+	Program = "syncthing"
 	Version = "unknown-dev"
-	Host    = "unknown" // Set by build script
-	User    = "unknown" // Set by build script
-	Stamp   = "0"       // Set by build script
+	Host    = "unknown"
+	User    = "unknown"
+	Stamp   = "0"
 
 	// Static
 	Codename = "Fermium Flea"
@@ -73,7 +74,7 @@ func setBuildData() {
 	Date = time.Unix(int64(stamp), 0)
 
 	date := Date.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`syncthing %s "%s" (%s %s-%s) %s@%s %s`, Version, Codename, runtime.Version(), runtime.GOOS, runtime.GOARCH, User, Host, date)
+	LongVersion = fmt.Sprintf(`%s %s "%s" (%s %s-%s) %s@%s %s`, Program, Version, Codename, runtime.Version(), runtime.GOOS, runtime.GOARCH, User, Host, date)
 
 	if len(Tags) > 0 {
 		LongVersion = fmt.Sprintf("%s [%s]", LongVersion, strings.Join(Tags, ", "))

lib/config/commit_test.go

@@ -44,7 +44,7 @@ func (validationError) String() string {
 func TestReplaceCommit(t *testing.T) {
 	t.Skip("broken, fails randomly, #3834")
 
-	w := Wrap("/dev/null", Configuration{Version: 0})
+	w := wrap("/dev/null", Configuration{Version: 0})
 	if w.RawCopy().Version != 0 {
 		t.Fatal("Config incorrect")
 	}

lib/config/config.go

@@ -10,6 +10,7 @@ package config
 import (
 	"encoding/json"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -86,11 +87,16 @@ var (
 		"stun.voiparound.com:3478",
 		"stun.voipbuster.com:3478",
 		"stun.voipstunt.com:3478",
-		"stun.voxgratia.org:3478",
 		"stun.xten.com:3478",
 	}
 )
 
+var (
+	errFolderIDEmpty     = errors.New("folder has empty ID")
+	errFolderIDDuplicate = errors.New("folder has duplicate ID")
+	errFolderPathEmpty   = errors.New("folder has empty path")
+)
+
 func New(myID protocol.DeviceID) Configuration {
 	var cfg Configuration
 	cfg.Version = CurrentVersion
@@ -102,7 +108,8 @@ func New(myID protocol.DeviceID) Configuration {
 
 	// Can't happen.
 	if err := cfg.prepare(myID); err != nil {
-		panic("bug: error in preparing new folder: " + err.Error())
+		l.Warnln("bug: error in preparing new folder:", err)
+		panic("error in preparing new folder")
 	}
 
 	return cfg
@@ -263,6 +270,16 @@ found:
 func (cfg *Configuration) clean() error {
 	util.FillNilSlices(&cfg.Options)
 
+	// Ensure that the device list is
+	// - free from duplicates
+	// - no devices with empty ID
+	// - sorted by ID
+	// Happen before preparting folders as that needs a correct device list.
+	cfg.Devices = ensureNoDuplicateOrEmptyIDDevices(cfg.Devices)
+	sort.Slice(cfg.Devices, func(a, b int) bool {
+		return cfg.Devices[a].DeviceID.Compare(cfg.Devices[b].DeviceID) == -1
+	})
+
 	// Prepare folders and check for duplicates. Duplicates are bad and
 	// dangerous, can't currently be resolved in the GUI, and shouldn't
 	// happen when configured by the GUI. We return with an error in that
@@ -273,12 +290,17 @@ func (cfg *Configuration) clean() error {
 		folder.prepare()
 
 		if folder.ID == "" {
-			return fmt.Errorf("folder with empty ID in configuration")
+			return errFolderIDEmpty
+		}
+
+		if folder.Path == "" {
+			return fmt.Errorf("folder %q: %v", folder.ID, errFolderPathEmpty)
 		}
 
 		if _, ok := existingFolders[folder.ID]; ok {
-			return fmt.Errorf("duplicate folder ID %q in configuration", folder.ID)
+			return fmt.Errorf("folder %q: %v", folder.ID, errFolderIDDuplicate)
 		}
+
 		existingFolders[folder.ID] = folder
 	}
 
@@ -298,14 +320,6 @@ func (cfg *Configuration) clean() error {
 		existingDevices[device.DeviceID] = true
 	}
 
-	// Ensure that the device list is
-	// - free from duplicates
-	// - sorted by ID
-	cfg.Devices = ensureNoDuplicateDevices(cfg.Devices)
-	sort.Slice(cfg.Devices, func(a, b int) bool {
-		return cfg.Devices[a].DeviceID.Compare(cfg.Devices[b].DeviceID) == -1
-	})
-
 	// Ensure that the folder list is sorted by ID
 	sort.Slice(cfg.Folders, func(a, b int) bool {
 		return cfg.Folders[a].ID < cfg.Folders[b].ID
@@ -464,14 +478,14 @@ loop:
 	return devices[0:count]
 }
 
-func ensureNoDuplicateDevices(devices []DeviceConfiguration) []DeviceConfiguration {
+func ensureNoDuplicateOrEmptyIDDevices(devices []DeviceConfiguration) []DeviceConfiguration {
 	count := len(devices)
 	i := 0
 	seenDevices := make(map[protocol.DeviceID]bool)
 loop:
 	for i < count {
 		id := devices[i].DeviceID
-		if _, ok := seenDevices[id]; ok {
+		if _, ok := seenDevices[id]; ok || id == protocol.EmptyDeviceID {
 			devices[i] = devices[count-1]
 			count--
 			continue loop

lib/config/config_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/d4l3k/messagediff"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
@@ -86,7 +87,7 @@ func TestDefaultValues(t *testing.T) {
 func TestDeviceConfig(t *testing.T) {
 	for i := OldestHandledVersion; i <= CurrentVersion; i++ {
 		os.RemoveAll(filepath.Join("testdata", DefaultMarkerName))
-		wr, err := Load(fmt.Sprintf("testdata/v%d.xml", i), device1)
+		wr, err := load(fmt.Sprintf("testdata/v%d.xml", i), device1)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -168,7 +169,7 @@ func TestDeviceConfig(t *testing.T) {
 }
 
 func TestNoListenAddresses(t *testing.T) {
-	cfg, err := Load("testdata/nolistenaddress.xml", device1)
+	cfg, err := load("testdata/nolistenaddress.xml", device1)
 	if err != nil {
 		t.Error(err)
 	}
@@ -225,7 +226,7 @@ func TestOverriddenValues(t *testing.T) {
 	}
 
 	os.Unsetenv("STNOUPGRADE")
-	cfg, err := Load("testdata/overridenvalues.xml", device1)
+	cfg, err := load("testdata/overridenvalues.xml", device1)
 	if err != nil {
 		t.Error(err)
 	}
@@ -270,7 +271,7 @@ func TestDeviceAddressesDynamic(t *testing.T) {
 		},
 	}
 
-	cfg, err := Load("testdata/deviceaddressesdynamic.xml", device4)
+	cfg, err := load("testdata/deviceaddressesdynamic.xml", device4)
 	if err != nil {
 		t.Error(err)
 	}
@@ -319,7 +320,7 @@ func TestDeviceCompression(t *testing.T) {
 		},
 	}
 
-	cfg, err := Load("testdata/devicecompression.xml", device4)
+	cfg, err := load("testdata/devicecompression.xml", device4)
 	if err != nil {
 		t.Error(err)
 	}
@@ -365,7 +366,7 @@ func TestDeviceAddressesStatic(t *testing.T) {
 		},
 	}
 
-	cfg, err := Load("testdata/deviceaddressesstatic.xml", device4)
+	cfg, err := load("testdata/deviceaddressesstatic.xml", device4)
 	if err != nil {
 		t.Error(err)
 	}
@@ -377,7 +378,7 @@ func TestDeviceAddressesStatic(t *testing.T) {
 }
 
 func TestVersioningConfig(t *testing.T) {
-	cfg, err := Load("testdata/versioningconfig.xml", device4)
+	cfg, err := load("testdata/versioningconfig.xml", device4)
 	if err != nil {
 		t.Error(err)
 	}
@@ -404,7 +405,7 @@ func TestIssue1262(t *testing.T) {
 		t.Skipf("path gets converted to absolute as part of the filesystem initialization on linux")
 	}
 
-	cfg, err := Load("testdata/issue-1262.xml", device4)
+	cfg, err := load("testdata/issue-1262.xml", device4)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -418,7 +419,7 @@ func TestIssue1262(t *testing.T) {
 }
 
 func TestIssue1750(t *testing.T) {
-	cfg, err := Load("testdata/issue-1750.xml", device4)
+	cfg, err := load("testdata/issue-1750.xml", device4)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -520,7 +521,7 @@ func TestNewSaveLoad(t *testing.T) {
 	}
 
 	intCfg := New(device1)
-	cfg := Wrap(path, intCfg)
+	cfg := wrap(path, intCfg)
 
 	// To make the equality pass later
 	cfg.(*wrapper).cfg.XMLName.Local = "configuration"
@@ -537,7 +538,7 @@ func TestNewSaveLoad(t *testing.T) {
 		t.Error(path, "does not exist")
 	}
 
-	cfg2, err := Load(path, device1)
+	cfg2, err := load(path, device1)
 	if err != nil {
 		t.Error(err)
 	}
@@ -564,7 +565,7 @@ func TestPrepare(t *testing.T) {
 }
 
 func TestCopy(t *testing.T) {
-	wrapper, err := Load("testdata/example.xml", device1)
+	wrapper, err := load("testdata/example.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -603,7 +604,7 @@ func TestCopy(t *testing.T) {
 }
 
 func TestPullOrder(t *testing.T) {
-	wrapper, err := Load("testdata/pullorder.xml", device1)
+	wrapper, err := load("testdata/pullorder.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -643,7 +644,7 @@ func TestPullOrder(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	wrapper = Wrap("testdata/pullorder.xml", cfg)
+	wrapper = wrap("testdata/pullorder.xml", cfg)
 	folders = wrapper.Folders()
 
 	for _, tc := range expected {
@@ -654,7 +655,7 @@ func TestPullOrder(t *testing.T) {
 }
 
 func TestLargeRescanInterval(t *testing.T) {
-	wrapper, err := Load("testdata/largeinterval.xml", device1)
+	wrapper, err := load("testdata/largeinterval.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -692,7 +693,7 @@ func TestGUIConfigURL(t *testing.T) {
 func TestDuplicateDevices(t *testing.T) {
 	// Duplicate devices should be removed
 
-	wrapper, err := Load("testdata/dupdevices.xml", device1)
+	wrapper, err := load("testdata/dupdevices.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -710,24 +711,20 @@ func TestDuplicateDevices(t *testing.T) {
 func TestDuplicateFolders(t *testing.T) {
 	// Duplicate folders are a loading error
 
-	_, err := Load("testdata/dupfolders.xml", device1)
-	if err == nil || !strings.HasPrefix(err.Error(), "duplicate folder ID") {
+	_, err := load("testdata/dupfolders.xml", device1)
+	if err == nil || !strings.Contains(err.Error(), errFolderIDDuplicate.Error()) {
 		t.Fatal(`Expected error to mention "duplicate folder ID":`, err)
 	}
 }
 
 func TestEmptyFolderPaths(t *testing.T) {
-	// Empty folder paths are allowed at the loading stage, and should not
+	// Empty folder paths are not allowed at the loading stage, and should not
 	// get messed up by the prepare steps (e.g., become the current dir or
 	// get a slash added so that it becomes the root directory or similar).
 
-	wrapper, err := Load("testdata/nopath.xml", device1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	folder := wrapper.Folders()["f1"]
-	if folder.cachedFilesystem != nil {
-		t.Errorf("Expected %q to be empty", folder.cachedFilesystem)
+	_, err := load("testdata/nopath.xml", device1)
+	if err == nil || !strings.Contains(err.Error(), errFolderPathEmpty.Error()) {
+		t.Fatal("Expected error due to empty folder path, got", err)
 	}
 }
 
@@ -794,7 +791,7 @@ func TestIgnoredDevices(t *testing.T) {
 	// Verify that ignored devices that are also present in the
 	// configuration are not in fact ignored.
 
-	wrapper, err := Load("testdata/ignoreddevices.xml", device1)
+	wrapper, err := load("testdata/ignoreddevices.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -812,7 +809,7 @@ func TestIgnoredFolders(t *testing.T) {
 	// configuration are not in fact ignored.
 	// Also, verify that folders that are shared with a device are not ignored.
 
-	wrapper, err := Load("testdata/ignoredfolders.xml", device1)
+	wrapper, err := load("testdata/ignoredfolders.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -848,7 +845,7 @@ func TestIgnoredFolders(t *testing.T) {
 func TestGetDevice(t *testing.T) {
 	// Verify that the Device() call does the right thing
 
-	wrapper, err := Load("testdata/ignoreddevices.xml", device1)
+	wrapper, err := load("testdata/ignoreddevices.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -875,7 +872,7 @@ func TestGetDevice(t *testing.T) {
 }
 
 func TestSharesRemovedOnDeviceRemoval(t *testing.T) {
-	wrapper, err := Load("testdata/example.xml", device1)
+	wrapper, err := load("testdata/example.xml", device1)
 	if err != nil {
 		t.Errorf("Failed: %s", err)
 	}
@@ -929,6 +926,7 @@ func TestIssue4219(t *testing.T) {
 		"folders": [
 			{
 				"id": "abcd123",
+				"path": "testdata",
 				"devices":[
 					{"deviceID": "GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY"}
 				]
@@ -959,7 +957,7 @@ func TestIssue4219(t *testing.T) {
 		t.Errorf("There should be three ignored folders, not %d", ignoredFolders)
 	}
 
-	w := Wrap("/tmp/cfg", cfg)
+	w := wrap("/tmp/cfg", cfg)
 	if !w.IgnoredFolder(device2, "t1") {
 		t.Error("Folder device2 t1 should be ignored")
 	}
@@ -1103,6 +1101,32 @@ func TestDeviceConfigObservedNotNil(t *testing.T) {
 	}
 }
 
+func TestRemoveDeviceWithEmptyID(t *testing.T) {
+	cfg := Configuration{
+		Devices: []DeviceConfiguration{
+			{
+				Name: "foo",
+			},
+		},
+		Folders: []FolderConfiguration{
+			{
+				ID:      "foo",
+				Path:    "testdata",
+				Devices: []FolderDeviceConfiguration{{}},
+			},
+		},
+	}
+
+	cfg.clean()
+
+	if len(cfg.Devices) != 0 {
+		t.Error("Expected device with empty ID to be removed from config:", cfg.Devices)
+	}
+	if len(cfg.Folders[0].Devices) != 0 {
+		t.Error("Expected device with empty ID to be removed from folder")
+	}
+}
+
 // defaultConfigAsMap returns a valid default config as a JSON-decoded
 // map[string]interface{}. This is useful to override random elements and
 // re-encode into JSON.
@@ -1122,3 +1146,11 @@ func defaultConfigAsMap() map[string]interface{} {
 	}
 	return tmp
 }
+
+func load(path string, myID protocol.DeviceID) (Wrapper, error) {
+	return Load(path, myID, events.NoopLogger)
+}
+
+func wrap(path string, cfg Configuration) Wrapper {
+	return Wrap(path, cfg, events.NoopLogger)
+}

lib/config/debug.go

@@ -7,16 +7,9 @@
 package config
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
 	l = logger.DefaultLogger.NewFacility("config", "Configuration loading and saving")
 )
-
-func init() {
-	l.SetDebug("config", strings.Contains(os.Getenv("STTRACE"), "config") || os.Getenv("STTRACE") == "all")
-}

lib/config/folderconfiguration.go

@@ -10,6 +10,10 @@ import (
 	"errors"
 	"fmt"
 	"runtime"
+	"strings"
+	"time"
+
+	"github.com/shirou/gopsutil/disk"
 
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
@@ -53,8 +57,10 @@ type FolderConfiguration struct {
 	WeakHashThresholdPct    int                         `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash.
 	MarkerName              string                      `xml:"markerName" json:"markerName"`
 	CopyOwnershipFromParent bool                        `xml:"copyOwnershipFromParent" json:"copyOwnershipFromParent"`
+	RawModTimeWindowS       int                         `xml:"modTimeWindowS" json:"modTimeWindowS"`
 
-	cachedFilesystem fs.Filesystem
+	cachedFilesystem    fs.Filesystem
+	cachedModTimeWindow time.Duration
 
 	DeprecatedReadOnly       bool    `xml:"ro,attr,omitempty" json:"-"`
 	DeprecatedMinDiskFreePct float64 `xml:"minDiskFreePct,omitempty" json:"-"`
@@ -92,7 +98,7 @@ func (f FolderConfiguration) Copy() FolderConfiguration {
 func (f FolderConfiguration) Filesystem() fs.Filesystem {
 	// This is intentionally not a pointer method, because things like
 	// cfg.Folders["default"].Filesystem() should be valid.
-	if f.cachedFilesystem == nil && f.Path != "" {
+	if f.cachedFilesystem == nil {
 		l.Infoln("bug: uncached filesystem call (should only happen in tests)")
 		return fs.NewFilesystem(f.FilesystemType, f.Path)
 	}
@@ -111,6 +117,10 @@ func (f FolderConfiguration) Versioner() versioner.Versioner {
 	return versionerFactory(f.ID, f.Filesystem(), f.Versioning.Params)
 }
 
+func (f FolderConfiguration) ModTimeWindow() time.Duration {
+	return f.cachedModTimeWindow
+}
+
 func (f *FolderConfiguration) CreateMarker() error {
 	if err := f.CheckPath(); err != ErrMarkerMissing {
 		return err
@@ -209,9 +219,7 @@ func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
 }
 
 func (f *FolderConfiguration) prepare() {
-	if f.Path != "" {
-		f.cachedFilesystem = fs.NewFilesystem(f.FilesystemType, f.Path)
-	}
+	f.cachedFilesystem = fs.NewFilesystem(f.FilesystemType, f.Path)
 
 	if f.RescanIntervalS > MaxRescanIntervalS {
 		f.RescanIntervalS = MaxRescanIntervalS
@@ -235,6 +243,21 @@ func (f *FolderConfiguration) prepare() {
 	if f.MarkerName == "" {
 		f.MarkerName = DefaultMarkerName
 	}
+
+	switch {
+	case f.RawModTimeWindowS > 0:
+		f.cachedModTimeWindow = time.Duration(f.RawModTimeWindowS) * time.Second
+	case runtime.GOOS == "android":
+		if usage, err := disk.Usage(f.Filesystem().URI()); err != nil {
+			f.cachedModTimeWindow = 2 * time.Second
+			l.Debugf(`Detecting FS at "%v" on android: Setting mtime window to 2s: err == "%v"`, f.Path, err)
+		} else if usage.Fstype == "" || strings.Contains(strings.ToLower(usage.Fstype), "fat") {
+			f.cachedModTimeWindow = 2 * time.Second
+			l.Debugf(`Detecting FS at "%v" on android: Setting mtime window to 2s: usage.Fstype == "%v"`, f.Path, usage.Fstype)
+		} else {
+			l.Debugf(`Detecting FS at %v on android: Leaving mtime window at 0: usage.Fstype == "%v"`, f.Path, usage.Fstype)
+		}
+	}
 }
 
 // RequiresRestartOnly returns a copy with only the attributes that require

lib/config/optionsconfiguration.go

@@ -57,6 +57,7 @@ type OptionsConfiguration struct {
 	StunKeepaliveStartS     int      `xml:"stunKeepaliveStartS" json:"stunKeepaliveStartS" default:"180"` // 0 for off
 	StunKeepaliveMinS       int      `xml:"stunKeepaliveMinS" json:"stunKeepaliveMinS" default:"20"`      // 0 for off
 	StunServers             []string `xml:"stunServer" json:"stunServers" default:"default"`
+	DatabaseTuning          Tuning   `xml:"databaseTuning" json:"databaseTuning" restart:"true"`
 
 	DeprecatedUPnPEnabled        bool     `xml:"upnpEnabled,omitempty" json:"-"`
 	DeprecatedUPnPLeaseM         int      `xml:"upnpLeaseMinutes,omitempty" json:"-"`

lib/config/size.go

@@ -87,11 +87,26 @@ func CheckFreeSpace(req Size, usage fs.Usage) error {
 	if req.Percentage() {
 		freePct := (float64(usage.Free) / float64(usage.Total)) * 100
 		if freePct < val {
-			return fmt.Errorf("%f %% < %v", freePct, req)
+			return fmt.Errorf("%.1f %% < %v", freePct, req)
 		}
 	} else if float64(usage.Free) < val {
-		return fmt.Errorf("%v < %v", usage.Free, req)
+		return fmt.Errorf("%sB < %v", formatSI(usage.Free), req)
 	}
 
 	return nil
 }
+
+func formatSI(b int64) string {
+	switch {
+	case b < 1000:
+		return fmt.Sprintf("%d ", b)
+	case b < 1000*1000:
+		return fmt.Sprintf("%.1f K", float64(b)/1000)
+	case b < 1000*1000*1000:
+		return fmt.Sprintf("%.1f M", float64(b)/(1000*1000))
+	case b < 1000*1000*1000*1000:
+		return fmt.Sprintf("%.1f G", float64(b)/(1000*1000*1000))
+	default:
+		return fmt.Sprintf("%.1f T", float64(b)/(1000*1000*1000*1000))
+	}
+}

lib/config/size_test.go

@@ -91,3 +91,42 @@ func TestParseSize(t *testing.T) {
 		}
 	}
 }
+
+func TestFormatSI(t *testing.T) {
+	cases := []struct {
+		bytes  int64
+		result string
+	}{
+		{
+			bytes:  0,
+			result: "0 ", // space for unit
+		},
+		{
+			bytes:  999,
+			result: "999 ",
+		},
+		{
+			bytes:  1000,
+			result: "1.0 K",
+		},
+		{
+			bytes:  1023 * 1000,
+			result: "1.0 M",
+		},
+		{
+			bytes:  5 * 1000 * 1000 * 1000,
+			result: "5.0 G",
+		},
+		{
+			bytes:  50000 * 1000 * 1000 * 1000 * 1000,
+			result: "50000.0 T",
+		},
+	}
+
+	for _, tc := range cases {
+		res := formatSI(tc.bytes)
+		if res != tc.result {
+			t.Errorf("formatSI(%d) => %q, expected %q", tc.bytes, res, tc.result)
+		}
+	}
+}

lib/config/testdata/dupdevices.xml

@@ -15,7 +15,7 @@
         <!-- duplicate, will be removed -->
         <address>192.0.2.5</address>
     </device>
-    <folder id="f2" directory="testdata/">
+    <folder id="f2" path="testdata/">
         <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
         <device id="GYRZZQBIRNPV4T7TC52WEQYJ3TFDQW6MWDFLMU4SSSU6EMFBK2VA"></device>
         <!-- duplicate device, will be removed -->

lib/config/testdata/dupfolders.xml

@@ -1,6 +1,6 @@
 <configuration version="15">
-    <folder id="f1" directory="testdata/">
+    <folder id="f1" path="testdata/">
     </folder>
-    <folder id="f1" directory="testdata/">
+    <folder id="f1" path="testdata/">
     </folder>
 </configuration>

lib/config/testdata/ignoredfolders.xml

@@ -8,7 +8,7 @@
         <ignoredFolder id="folder1"/>
         <ignoredFolder id="folder2"/>
     </device>
-    <folder id="folder1" directory="testdata/">
+    <folder id="folder1" path="testdata/">
         <device id="GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY"></device>
     </folder>
 </configuration>

lib/config/testdata/pullorder.xml

@@ -1,25 +1,25 @@
 <configuration version="10">
-    <folder id="f1" directory="testdata/">
+    <folder id="f1" path="testdata/">
     </folder>
-    <folder id="f2" directory="testdata/">
+    <folder id="f2" path="testdata/">
         <order>random</order>
     </folder>
-    <folder id="f3" directory="testdata/">
+    <folder id="f3" path="testdata/">
         <order>alphabetic</order>
     </folder>
-    <folder id="f4" directory="testdata/">
+    <folder id="f4" path="testdata/">
         <order>whatever</order>
     </folder>
-    <folder id="f5" directory="testdata/">
+    <folder id="f5" path="testdata/">
         <order>smallestFirst</order>
     </folder>
-    <folder id="f6" directory="testdata/">
+    <folder id="f6" path="testdata/">
         <order>largestFirst</order>
     </folder>
-    <folder id="f7" directory="testdata/">
+    <folder id="f7" path="testdata/">
         <order>oldestFirst</order>
     </folder>
-    <folder id="f8" directory="testdata/">
+    <folder id="f8" path="testdata/">
         <order>newestFirst</order>
     </folder>
 </configuration>

lib/config/tuning.go

@@ -0,0 +1,47 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package config
+
+type Tuning int
+
+const (
+	// N.b. these constants must match those in lib/db.Tuning!
+	TuningAuto Tuning = iota // default is auto
+	TuningSmall
+	TuningLarge
+)
+
+func (t Tuning) String() string {
+	switch t {
+	case TuningAuto:
+		return "auto"
+	case TuningSmall:
+		return "small"
+	case TuningLarge:
+		return "large"
+	default:
+		return "unknown"
+	}
+}
+
+func (t Tuning) MarshalText() ([]byte, error) {
+	return []byte(t.String()), nil
+}
+
+func (t *Tuning) UnmarshalText(bs []byte) error {
+	switch string(bs) {
+	case "auto":
+		*t = TuningAuto
+	case "small":
+		*t = TuningSmall
+	case "large":
+		*t = TuningLarge
+	default:
+		*t = TuningAuto
+	}
+	return nil
+}

lib/config/tuning_test.go

@@ -0,0 +1,26 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package config_test
+
+import (
+	"testing"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/db"
+)
+
+func TestTuningMatches(t *testing.T) {
+	if int(config.TuningAuto) != int(db.TuningAuto) {
+		t.Error("mismatch for TuningAuto")
+	}
+	if int(config.TuningSmall) != int(db.TuningSmall) {
+		t.Error("mismatch for TuningSmall")
+	}
+	if int(config.TuningLarge) != int(db.TuningLarge) {
+		t.Error("mismatch for TuningLarge")
+	}
+}

lib/config/wrapper.go

@@ -96,8 +96,9 @@ type Wrapper interface {
 }
 
 type wrapper struct {
-	cfg  Configuration
-	path string
+	cfg      Configuration
+	path     string
+	evLogger events.Logger
 
 	deviceMap map[protocol.DeviceID]DeviceConfiguration
 	folderMap map[string]FolderConfiguration
@@ -133,18 +134,19 @@ func (w *wrapper) StunServers() []string {
 
 // Wrap wraps an existing Configuration structure and ties it to a file on
 // disk.
-func Wrap(path string, cfg Configuration) Wrapper {
+func Wrap(path string, cfg Configuration, evLogger events.Logger) Wrapper {
 	w := &wrapper{
-		cfg:  cfg,
-		path: path,
-		mut:  sync.NewMutex(),
+		cfg:      cfg,
+		path:     path,
+		evLogger: evLogger,
+		mut:      sync.NewMutex(),
 	}
 	return w
 }
 
 // Load loads an existing file on disk and returns a new configuration
 // wrapper.
-func Load(path string, myID protocol.DeviceID) (Wrapper, error) {
+func Load(path string, myID protocol.DeviceID, evLogger events.Logger) (Wrapper, error) {
 	fd, err := os.Open(path)
 	if err != nil {
 		return nil, err
@@ -156,7 +158,7 @@ func Load(path string, myID protocol.DeviceID) (Wrapper, error) {
 		return nil, err
 	}
 
-	return Wrap(path, cfg), nil
+	return Wrap(path, cfg, evLogger), nil
 }
 
 func (w *wrapper) ConfigPath() string {
@@ -450,7 +452,7 @@ func (w *wrapper) Save() error {
 		return err
 	}
 
-	events.Default.Log(events.ConfigSaved, w.cfg)
+	w.evLogger.Log(events.ConfigSaved, w.cfg)
 	return nil
 }
 
@@ -501,8 +503,6 @@ func (w *wrapper) MyName() string {
 }
 
 func (w *wrapper) AddOrUpdatePendingDevice(device protocol.DeviceID, name, address string) {
-	defer w.Save()
-
 	w.mut.Lock()
 	defer w.mut.Unlock()
 
@@ -524,8 +524,6 @@ func (w *wrapper) AddOrUpdatePendingDevice(device protocol.DeviceID, name, addre
 }
 
 func (w *wrapper) AddOrUpdatePendingFolder(id, label string, device protocol.DeviceID) {
-	defer w.Save()
-
 	w.mut.Lock()
 	defer w.mut.Unlock()
 

lib/connections/debug.go

@@ -7,16 +7,9 @@
 package connections
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
 	l = logger.DefaultLogger.NewFacility("connections", "Connection handling")
 )
-
-func init() {
-	l.SetDebug("connections", strings.Contains(os.Getenv("STTRACE"), "connections") || os.Getenv("STTRACE") == "all")
-}

lib/connections/lan_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
 )
 
 func TestIsLANHost(t *testing.T) {
@@ -35,7 +36,7 @@ func TestIsLANHost(t *testing.T) {
 		Options: config.OptionsConfiguration{
 			AlwaysLocalNets: []string{"10.20.30.0/24"},
 		},
-	})
+	}, events.NoopLogger)
 	s := &service{cfg: cfg}
 
 	for _, tc := range cases {

lib/connections/limiter.go

@@ -32,9 +32,13 @@ type limiter struct {
 type waiter interface {
 	// This is the rate limiting operation
 	WaitN(ctx context.Context, n int) error
+	Limit() rate.Limit
 }
 
-const limiterBurstSize = 4 * 128 << 10
+const (
+	limiterBurstSize   = 4 * 128 << 10
+	maxSingleWriteSize = 8 << 10
+)
 
 func newLimiter(cfg config.Wrapper) *limiter {
 	l := &limiter{
@@ -186,19 +190,23 @@ func (lim *limiter) getLimiters(remoteID protocol.DeviceID, rw io.ReadWriter, is
 
 func (lim *limiter) newLimitedReaderLocked(remoteID protocol.DeviceID, r io.Reader, isLAN bool) io.Reader {
 	return &limitedReader{
-		reader:    r,
-		limitsLAN: &lim.limitsLAN,
-		waiter:    totalWaiter{lim.getReadLimiterLocked(remoteID), lim.read},
-		isLAN:     isLAN,
+		reader: r,
+		waiterHolder: waiterHolder{
+			waiter:    totalWaiter{lim.getReadLimiterLocked(remoteID), lim.read},
+			limitsLAN: &lim.limitsLAN,
+			isLAN:     isLAN,
+		},
 	}
 }
 
 func (lim *limiter) newLimitedWriterLocked(remoteID protocol.DeviceID, w io.Writer, isLAN bool) io.Writer {
 	return &limitedWriter{
-		writer:    w,
-		limitsLAN: &lim.limitsLAN,
-		waiter:    totalWaiter{lim.getWriteLimiterLocked(remoteID), lim.write},
-		isLAN:     isLAN,
+		writer: w,
+		waiterHolder: waiterHolder{
+			waiter:    totalWaiter{lim.getWriteLimiterLocked(remoteID), lim.write},
+			limitsLAN: &lim.limitsLAN,
+			isLAN:     isLAN,
+		},
 	}
 }
 
@@ -221,53 +229,87 @@ func getRateLimiter(m map[protocol.DeviceID]*rate.Limiter, deviceID protocol.Dev
 
 // limitedReader is a rate limited io.Reader
 type limitedReader struct {
-	reader    io.Reader
-	limitsLAN *atomicBool
-	waiter    waiter
-	isLAN     bool
+	reader io.Reader
+	waiterHolder
 }
 
 func (r *limitedReader) Read(buf []byte) (int, error) {
 	n, err := r.reader.Read(buf)
-	if !r.isLAN || r.limitsLAN.get() {
-		take(r.waiter, n)
+	if !r.unlimited() {
+		r.take(n)
 	}
 	return n, err
 }
 
 // limitedWriter is a rate limited io.Writer
 type limitedWriter struct {
-	writer    io.Writer
-	limitsLAN *atomicBool
-	waiter    waiter
-	isLAN     bool
+	writer io.Writer
+	waiterHolder
 }
 
 func (w *limitedWriter) Write(buf []byte) (int, error) {
-	if !w.isLAN || w.limitsLAN.get() {
-		take(w.waiter, len(buf))
+	if w.unlimited() {
+		return w.writer.Write(buf)
 	}
-	return w.writer.Write(buf)
+
+	// This does (potentially) multiple smaller writes in order to be less
+	// bursty with large writes and slow rates.
+	written := 0
+	for written < len(buf) {
+		toWrite := maxSingleWriteSize
+		if toWrite > len(buf)-written {
+			toWrite = len(buf) - written
+		}
+		w.take(toWrite)
+		n, err := w.writer.Write(buf[written : written+toWrite])
+		written += n
+		if err != nil {
+			return written, err
+		}
+	}
+
+	return written, nil
 }
 
-// take is a utility function to consume tokens from a overall rate.Limiter and deviceLimiter.
-// No call to WaitN can be larger than the limiter burst size so we split it up into
-// several calls when necessary.
-func take(waiter waiter, tokens int) {
+// waiterHolder is the common functionality around having and evaluating a
+// waiter, valid for both writers and readers
+type waiterHolder struct {
+	waiter    waiter
+	limitsLAN *atomicBool
+	isLAN     bool
+}
+
+// unlimited returns true if the waiter is not limiting the rate
+func (w waiterHolder) unlimited() bool {
+	if w.isLAN && !w.limitsLAN.get() {
+		return true
+	}
+	return w.waiter.Limit() == rate.Inf
+}
+
+// take is a utility function to consume tokens, because no call to WaitN
+// must be larger than the limiter burst size or it will hang.
+func (w waiterHolder) take(tokens int) {
+	// For writes we already split the buffer into smaller operations so those
+	// will always end up in the fast path below. For reads, however, we don't
+	// control the size of the incoming buffer and don't split the calls
+	// into the lower level reads so we might get a large amount of data and
+	// end up in the loop further down.
+
 	if tokens < limiterBurstSize {
-		// This is the by far more common case so we get it out of the way
-		// early.
-		waiter.WaitN(context.TODO(), tokens)
+		// Fast path. We won't get an error from WaitN as we don't pass a
+		// context with a deadline.
+		_ = w.waiter.WaitN(context.TODO(), tokens)
 		return
 	}
 
 	for tokens > 0 {
 		// Consume limiterBurstSize tokens at a time until we're done.
 		if tokens > limiterBurstSize {
-			waiter.WaitN(context.TODO(), limiterBurstSize)
+			_ = w.waiter.WaitN(context.TODO(), limiterBurstSize)
 			tokens -= limiterBurstSize
 		} else {
-			waiter.WaitN(context.TODO(), tokens)
+			_ = w.waiter.WaitN(context.TODO(), tokens)
 			tokens = 0
 		}
 	}
@@ -300,3 +342,13 @@ func (tw totalWaiter) WaitN(ctx context.Context, n int) error {
 	}
 	return nil
 }
+
+func (tw totalWaiter) Limit() rate.Limit {
+	min := rate.Inf
+	for _, w := range tw {
+		if l := w.Limit(); l < min {
+			min = l
+		}
+	}
+	return min
+}

lib/connections/limiter_test.go

@@ -7,11 +7,16 @@
 package connections
 
 import (
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"golang.org/x/time/rate"
+	"bytes"
+	crand "crypto/rand"
+	"io"
 	"math/rand"
 	"testing"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"golang.org/x/time/rate"
 )
 
 var device1, device2, device3, device4 protocol.DeviceID
@@ -25,7 +30,7 @@ func init() {
 }
 
 func initConfig() config.Wrapper {
-	cfg := config.Wrap("/dev/null", config.New(device1))
+	cfg := config.Wrap("/dev/null", config.New(device1), events.NoopLogger)
 	dev1Conf = config.NewDeviceConfiguration(device1, "device1")
 	dev2Conf = config.NewDeviceConfiguration(device2, "device2")
 	dev3Conf = config.NewDeviceConfiguration(device3, "device3")
@@ -184,6 +189,151 @@ func TestAddAndRemove(t *testing.T) {
 	checkActualAndExpected(t, actualR, actualW, expectedR, expectedW)
 }
 
+func TestLimitedWriterWrite(t *testing.T) {
+	// Check that the limited writer writes the correct data in the correct manner.
+
+	// A buffer with random data that is larger than the write size and not
+	// a precise multiple either.
+	src := make([]byte, int(12.5*maxSingleWriteSize))
+	if _, err := crand.Reader.Read(src); err != nil {
+		t.Fatal(err)
+	}
+
+	// Write it to the destination using a limited writer, with a wrapper to
+	// count the write calls. The defaults on the limited writer should mean
+	// it is used (and doesn't take the fast path). In practice the limiter
+	// won't delay the test as the burst size is large enough to accommodate
+	// regardless of the rate.
+	dst := new(bytes.Buffer)
+	cw := &countingWriter{w: dst}
+	lw := &limitedWriter{
+		writer: cw,
+		waiterHolder: waiterHolder{
+			waiter:    rate.NewLimiter(rate.Limit(42), limiterBurstSize),
+			limitsLAN: new(atomicBool),
+			isLAN:     false, // enables limiting
+		},
+	}
+	if _, err := io.Copy(lw, bytes.NewReader(src)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify there were lots of writes and that the end result is identical.
+	if cw.writeCount != 13 {
+		t.Error("expected lots of smaller writes, but not too many")
+	}
+	if !bytes.Equal(src, dst.Bytes()) {
+		t.Error("results should be equal")
+	}
+
+	// Write it to the destination using a limited writer, with a wrapper to
+	// count the write calls. Now we make sure the fast path is used.
+	dst = new(bytes.Buffer)
+	cw = &countingWriter{w: dst}
+	lw = &limitedWriter{
+		writer: cw,
+		waiterHolder: waiterHolder{
+			waiter:    rate.NewLimiter(rate.Limit(42), limiterBurstSize),
+			limitsLAN: new(atomicBool),
+			isLAN:     true, // disables limiting
+		},
+	}
+	if _, err := io.Copy(lw, bytes.NewReader(src)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify there were a single write and that the end result is identical.
+	if cw.writeCount != 1 {
+		t.Error("expected just the one write")
+	}
+	if !bytes.Equal(src, dst.Bytes()) {
+		t.Error("results should be equal")
+	}
+
+	// Once more, but making sure the fast path is used for an unlimited
+	// rate, with multiple unlimited raters even (global and per-device).
+	dst = new(bytes.Buffer)
+	cw = &countingWriter{w: dst}
+	lw = &limitedWriter{
+		writer: cw,
+		waiterHolder: waiterHolder{
+			waiter:    totalWaiter{rate.NewLimiter(rate.Inf, limiterBurstSize), rate.NewLimiter(rate.Inf, limiterBurstSize)},
+			limitsLAN: new(atomicBool),
+			isLAN:     false, // enables limiting
+		},
+	}
+	if _, err := io.Copy(lw, bytes.NewReader(src)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify there were a single write and that the end result is identical.
+	if cw.writeCount != 1 {
+		t.Error("expected just the one write")
+	}
+	if !bytes.Equal(src, dst.Bytes()) {
+		t.Error("results should be equal")
+	}
+
+	// Once more, but making sure we *don't* take the fast path when there
+	// is a combo of limited and unlimited writers.
+	dst = new(bytes.Buffer)
+	cw = &countingWriter{w: dst}
+	lw = &limitedWriter{
+		writer: cw,
+		waiterHolder: waiterHolder{
+			waiter: totalWaiter{
+				rate.NewLimiter(rate.Inf, limiterBurstSize),
+				rate.NewLimiter(rate.Limit(42), limiterBurstSize),
+				rate.NewLimiter(rate.Inf, limiterBurstSize),
+			},
+			limitsLAN: new(atomicBool),
+			isLAN:     false, // enables limiting
+		},
+	}
+	if _, err := io.Copy(lw, bytes.NewReader(src)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify there were lots of writes and that the end result is identical.
+	if cw.writeCount != 13 {
+		t.Error("expected just the one write")
+	}
+	if !bytes.Equal(src, dst.Bytes()) {
+		t.Error("results should be equal")
+	}
+}
+
+func TestTotalWaiterLimit(t *testing.T) {
+	cases := []struct {
+		w waiter
+		r rate.Limit
+	}{
+		{
+			totalWaiter{},
+			rate.Inf,
+		},
+		{
+			totalWaiter{rate.NewLimiter(rate.Inf, 42)},
+			rate.Inf,
+		},
+		{
+			totalWaiter{rate.NewLimiter(rate.Inf, 42), rate.NewLimiter(rate.Inf, 42)},
+			rate.Inf,
+		},
+		{
+			totalWaiter{rate.NewLimiter(rate.Inf, 42), rate.NewLimiter(rate.Limit(12), 42), rate.NewLimiter(rate.Limit(15), 42)},
+			rate.Limit(12),
+		},
+	}
+
+	for _, tc := range cases {
+		l := tc.w.Limit()
+		if l != tc.r {
+			t.Error("incorrect limit returned")
+		}
+	}
+}
+
 func checkActualAndExpected(t *testing.T, actualR, actualW, expectedR, expectedW map[protocol.DeviceID]*rate.Limiter) {
 	t.Helper()
 	if len(expectedW) != len(actualW) || len(expectedR) != len(actualR) {
@@ -203,3 +353,13 @@ func checkActualAndExpected(t *testing.T, actualR, actualW, expectedR, expectedW
 		}
 	}
 }
+
+type countingWriter struct {
+	w          io.Writer
+	writeCount int
+}
+
+func (w *countingWriter) Write(data []byte) (int, error) {
+	w.writeCount++
+	return w.w.Write(data)
+}

lib/connections/quic_dial.go

@@ -11,6 +11,7 @@ package connections
 import (
 	"context"
 	"crypto/tls"
+	"fmt"
 	"net"
 	"net/url"
 	"time"
@@ -22,7 +23,13 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-const quicPriority = 100
+const (
+	quicPriority = 100
+
+	// The timeout for connecting, accepting and creating the various
+	// streams.
+	quicOperationTimeout = 10 * time.Second
+)
 
 func init() {
 	factory := &quicDialerFactory{}
@@ -36,7 +43,7 @@ type quicDialer struct {
 	tlsCfg *tls.Config
 }
 
-func (d *quicDialer) Dial(id protocol.DeviceID, uri *url.URL) (internalConn, error) {
+func (d *quicDialer) Dial(_ protocol.DeviceID, uri *url.URL) (internalConn, error) {
 	uri = fixupPort(uri, config.DefaultQUICPort)
 
 	addr, err := net.ResolveUDPAddr("udp", uri.Host)
@@ -60,38 +67,25 @@ func (d *quicDialer) Dial(id protocol.DeviceID, uri *url.URL) (internalConn, err
 		}
 	}
 
-	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), quicOperationTimeout)
+	defer cancel()
+
 	session, err := quic.DialContext(ctx, conn, addr, uri.Host, d.tlsCfg, quicConfig)
 	if err != nil {
 		if createdConn != nil {
 			_ = createdConn.Close()
 		}
-		return internalConn{}, err
+		return internalConn{}, fmt.Errorf("dial: %v", err)
 	}
 
-	// OpenStreamSync is blocks, but we want to make sure the connection is usable
-	// before we start killing off other connections, so do the dance.
-	ok := make(chan struct{})
-	go func() {
-		select {
-		case <-ok:
-			return
-		case <-time.After(10 * time.Second):
-			l.Debugln("timed out waiting for OpenStream on", session.RemoteAddr())
-			// This will unblock OpenStreamSync
-			_ = session.Close()
-		}
-	}()
-
-	stream, err := session.OpenStreamSync()
-	close(ok)
+	stream, err := session.OpenStreamSync(ctx)
 	if err != nil {
 		// It's ok to close these, this does not close the underlying packetConn.
 		_ = session.Close()
 		if createdConn != nil {
 			_ = createdConn.Close()
 		}
-		return internalConn{}, err
+		return internalConn{}, fmt.Errorf("open stream: %v", err)
 	}
 
 	return internalConn{&quicTlsConn{session, stream, createdConn}, connTypeQUICClient, quicPriority}, nil

lib/connections/quic_listen.go

@@ -9,6 +9,7 @@
 package connections
 
 import (
+	"context"
 	"crypto/tls"
 	"net"
 	"net/url"
@@ -23,6 +24,7 @@ import (
 	"github.com/syncthing/syncthing/lib/connections/registry"
 	"github.com/syncthing/syncthing/lib/nat"
 	"github.com/syncthing/syncthing/lib/stun"
+	"github.com/syncthing/syncthing/lib/util"
 )
 
 func init() {
@@ -33,6 +35,7 @@ func init() {
 }
 
 type quicListener struct {
+	util.ServiceWithError
 	nat atomic.Value
 
 	onAddressesChangedNotifier
@@ -40,12 +43,10 @@ type quicListener struct {
 	uri     *url.URL
 	cfg     config.Wrapper
 	tlsCfg  *tls.Config
-	stop    chan struct{}
 	conns   chan internalConn
 	factory listenerFactory
 
 	address *url.URL
-	err     error
 	mut     sync.Mutex
 }
 
@@ -77,20 +78,17 @@ func (t *quicListener) OnExternalAddressChanged(address *stun.Host, via string)
 	}
 }
 
-func (t *quicListener) Serve() {
-	t.mut.Lock()
-	t.err = nil
-	t.mut.Unlock()
-
+func (t *quicListener) serve(stop chan struct{}) error {
 	network := strings.Replace(t.uri.Scheme, "quic", "udp", -1)
 
+	// Convert the stop channel into a context
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() { <-stop; cancel() }()
+
 	packetConn, err := net.ListenPacket(network, t.uri.Host)
 	if err != nil {
-		t.mut.Lock()
-		t.err = err
-		t.mut.Unlock()
 		l.Infoln("Listen (BEP/quic):", err)
-		return
+		return err
 	}
 	defer func() { _ = packetConn.Close() }()
 
@@ -105,63 +103,52 @@ func (t *quicListener) Serve() {
 
 	listener, err := quic.Listen(conn, t.tlsCfg, quicConfig)
 	if err != nil {
-		t.mut.Lock()
-		t.err = err
-		t.mut.Unlock()
 		l.Infoln("Listen (BEP/quic):", err)
-		return
+		return err
 	}
+	defer listener.Close()
 
 	l.Infof("QUIC listener (%v) starting", packetConn.LocalAddr())
 	defer l.Infof("QUIC listener (%v) shutting down", packetConn.LocalAddr())
 
-	// Accept is forever, so handle stops externally.
-	go func() {
-		select {
-		case <-t.stop:
-			_ = listener.Close()
-		}
-	}()
+	acceptFailures := 0
+	const maxAcceptFailures = 10
 
 	for {
-		// Blocks forever, see https://github.com/lucas-clemente/quic-go/issues/1915
-		session, err := listener.Accept()
-
 		select {
-		case <-t.stop:
-			if err == nil {
-				_ = session.Close()
-			}
-			return
+		case <-ctx.Done():
+			return nil
 		default:
 		}
-		if err != nil {
-			if err, ok := err.(net.Error); !ok || !err.Timeout() {
-				l.Warnln("Listen (BEP/quic): Accepting connection:", err)
+
+		session, err := listener.Accept(ctx)
+		if err == context.Canceled {
+			return nil
+		} else if err != nil {
+			l.Infoln("Listen (BEP/quic): Accepting connection:", err)
+
+			acceptFailures++
+			if acceptFailures > maxAcceptFailures {
+				// Return to restart the listener, because something
+				// seems permanently damaged.
+				return err
 			}
+
+			// Slightly increased delay for each failure.
+			time.Sleep(time.Duration(acceptFailures) * time.Second)
+
 			continue
 		}
 
+		acceptFailures = 0
+
 		l.Debugln("connect from", session.RemoteAddr())
 
-		// Accept blocks forever, give it 10s to do it's thing.
-		ok := make(chan struct{})
-		go func() {
-			select {
-			case <-ok:
-				return
-			case <-t.stop:
-				_ = session.Close()
-			case <-time.After(10 * time.Second):
-				l.Debugln("timed out waiting for AcceptStream on", session.RemoteAddr())
-				_ = session.Close()
-			}
-		}()
-
-		stream, err := session.AcceptStream()
-		close(ok)
+		streamCtx, cancel := context.WithTimeout(ctx, quicOperationTimeout)
+		stream, err := session.AcceptStream(streamCtx)
+		cancel()
 		if err != nil {
-			l.Debugln("failed to accept stream from", session.RemoteAddr(), err.Error())
+			l.Debugf("failed to accept stream from %s: %v", session.RemoteAddr(), err)
 			_ = session.Close()
 			continue
 		}
@@ -170,10 +157,6 @@ func (t *quicListener) Serve() {
 	}
 }
 
-func (t *quicListener) Stop() {
-	close(t.stop)
-}
-
 func (t *quicListener) URI() *url.URL {
 	return t.uri
 }
@@ -192,13 +175,6 @@ func (t *quicListener) LANAddresses() []*url.URL {
 	return []*url.URL{t.uri}
 }
 
-func (t *quicListener) Error() error {
-	t.mut.Lock()
-	err := t.err
-	t.mut.Unlock()
-	return err
-}
-
 func (t *quicListener) String() string {
 	return t.uri.String()
 }
@@ -227,9 +203,9 @@ func (f *quicListenerFactory) New(uri *url.URL, cfg config.Wrapper, tlsCfg *tls.
 		cfg:     cfg,
 		tlsCfg:  tlsCfg,
 		conns:   conns,
-		stop:    make(chan struct{}),
 		factory: f,
 	}
+	l.ServiceWithError = util.AsServiceWithError(l.serve)
 	l.nat.Store(stun.NATUnknown)
 	return l
 }

lib/connections/relay_listen.go

@@ -16,6 +16,7 @@ import (
 	"github.com/syncthing/syncthing/lib/dialer"
 	"github.com/syncthing/syncthing/lib/nat"
 	"github.com/syncthing/syncthing/lib/relay/client"
+	"github.com/syncthing/syncthing/lib/util"
 )
 
 func init() {
@@ -26,6 +27,7 @@ func init() {
 }
 
 type relayListener struct {
+	util.ServiceWithError
 	onAddressesChangedNotifier
 
 	uri     *url.URL
@@ -34,30 +36,22 @@ type relayListener struct {
 	conns   chan internalConn
 	factory listenerFactory
 
-	err    error
 	client client.RelayClient
 	mut    sync.RWMutex
 }
 
-func (t *relayListener) Serve() {
-	t.mut.Lock()
-	t.err = nil
-	t.mut.Unlock()
-
+func (t *relayListener) serve(stop chan struct{}) error {
 	clnt, err := client.NewClient(t.uri, t.tlsCfg.Certificates, nil, 10*time.Second)
-	invitations := clnt.Invitations()
 	if err != nil {
-		t.mut.Lock()
-		t.err = err
-		t.mut.Unlock()
-		l.Warnln("Listen (BEP/relay):", err)
-		return
+		l.Infoln("Listen (BEP/relay):", err)
+		return err
 	}
-
-	go clnt.Serve()
+	invitations := clnt.Invitations()
 
 	t.mut.Lock()
 	t.client = clnt
+	go clnt.Serve()
+	defer clnt.Stop()
 	t.mut.Unlock()
 
 	oldURI := clnt.URI()
@@ -69,7 +63,10 @@ func (t *relayListener) Serve() {
 		select {
 		case inv, ok := <-invitations:
 			if !ok {
-				return
+				if err := clnt.Error(); err != nil {
+					l.Infoln("Listen (BEP/relay):", err)
+				}
+				return err
 			}
 
 			conn, err := client.JoinSession(inv)
@@ -114,18 +111,13 @@ func (t *relayListener) Serve() {
 				oldURI = currentURI
 				t.notifyAddressesChanged(t)
 			}
+
+		case <-stop:
+			return nil
 		}
 	}
 }
 
-func (t *relayListener) Stop() {
-	t.mut.RLock()
-	if t.client != nil {
-		t.client.Stop()
-	}
-	t.mut.RUnlock()
-}
-
 func (t *relayListener) URI() *url.URL {
 	return t.uri
 }
@@ -152,18 +144,16 @@ func (t *relayListener) LANAddresses() []*url.URL {
 }
 
 func (t *relayListener) Error() error {
-	t.mut.RLock()
-	err := t.err
-	var cerr error
-	if t.client != nil {
-		cerr = t.client.Error()
-	}
-	t.mut.RUnlock()
-
+	err := t.ServiceWithError.Error()
 	if err != nil {
 		return err
 	}
-	return cerr
+	t.mut.RLock()
+	defer t.mut.RUnlock()
+	if t.client != nil {
+		return t.client.Error()
+	}
+	return nil
 }
 
 func (t *relayListener) Factory() listenerFactory {
@@ -181,13 +171,15 @@ func (t *relayListener) NATType() string {
 type relayListenerFactory struct{}
 
 func (f *relayListenerFactory) New(uri *url.URL, cfg config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {
-	return &relayListener{
+	t := &relayListener{
 		uri:     uri,
 		cfg:     cfg,
 		tlsCfg:  tlsCfg,
 		conns:   conns,
 		factory: f,
 	}
+	t.ServiceWithError = util.AsServiceWithError(t.serve)
+	return t
 }
 
 func (relayListenerFactory) Valid(cfg config.Configuration) error {

lib/connections/service.go

@@ -83,13 +83,14 @@ var tlsCipherSuiteNames = map[uint16]string{
 
 var tlsVersionNames = map[uint16]string{
 	tls.VersionTLS12: "TLS1.2",
-	772:              "TLS1.3", // tls.VersionTLS13 constant available in Go 1.12+
+	tls.VersionTLS13: "TLS1.3",
 }
 
 // Service listens and dials all configured unconnected devices, via supported
 // dialers. Successful connections are handed to the model.
 type Service interface {
 	suture.Service
+	discover.AddressLister
 	ListenerStatus() map[string]ListenerStatusEntry
 	ConnectionStatus() map[string]ConnectionStatusEntry
 	NATType() string
@@ -119,6 +120,7 @@ type service struct {
 	limiter              *limiter
 	natService           *nat.Service
 	natServiceToken      *suture.ServiceToken
+	evLogger             events.Logger
 
 	listenersMut       sync.RWMutex
 	listeners          map[string]genericListener
@@ -129,9 +131,7 @@ type service struct {
 	connectionStatus    map[string]ConnectionStatusEntry // address -> latest error/status
 }
 
-func NewService(cfg config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *tls.Config, discoverer discover.Finder,
-	bepProtocolName string, tlsDefaultCommonName string) *service {
-
+func NewService(cfg config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *tls.Config, discoverer discover.Finder, bepProtocolName string, tlsDefaultCommonName string, evLogger events.Logger) Service {
 	service := &service{
 		Supervisor: suture.New("connections.Service", suture.Spec{
 			Log: func(line string) {
@@ -149,6 +149,7 @@ func NewService(cfg config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *t
 		tlsDefaultCommonName: tlsDefaultCommonName,
 		limiter:              newLimiter(cfg),
 		natService:           nat.NewService(myID, cfg),
+		evLogger:             evLogger,
 
 		listenersMut:   sync.NewRWMutex(),
 		listeners:      make(map[string]genericListener),
@@ -184,16 +185,22 @@ func NewService(cfg config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *t
 	// the common handling regardless of whether the connection was
 	// incoming or outgoing.
 
-	service.Add(serviceFunc(service.connect))
-	service.Add(serviceFunc(service.handle))
+	service.Add(util.AsService(service.connect))
+	service.Add(util.AsService(service.handle))
 	service.Add(service.listenerSupervisor)
 
 	return service
 }
 
-func (s *service) handle() {
-next:
-	for c := range s.conns {
+func (s *service) handle(stop chan struct{}) {
+	var c internalConn
+	for {
+		select {
+		case <-stop:
+			return
+		case c = <-s.conns:
+		}
+
 		cs := c.ConnectionState()
 
 		// We should have negotiated the next level protocol "bep/1.0" as part
@@ -225,7 +232,7 @@ next:
 			continue
 		}
 
-		c.SetDeadline(time.Now().Add(20 * time.Second))
+		_ = c.SetDeadline(time.Now().Add(20 * time.Second))
 		hello, err := protocol.ExchangeHello(c, s.model.GetHello(remoteID))
 		if err != nil {
 			if protocol.IsVersionMismatch(err) {
@@ -249,7 +256,7 @@ next:
 			c.Close()
 			continue
 		}
-		c.SetDeadline(time.Time{})
+		_ = c.SetDeadline(time.Time{})
 
 		// The Model will return an error for devices that we don't want to
 		// have a connection with for whatever reason, for example unknown devices.
@@ -298,7 +305,7 @@ next:
 			// config. Warn instead of Info.
 			l.Warnf("Bad certificate from %s at %s: %v", remoteID, c, err)
 			c.Close()
-			continue next
+			continue
 		}
 
 		// Wrap the connection in rate limiters. The limiter itself will
@@ -313,11 +320,11 @@ next:
 		l.Infof("Established secure connection to %s at %s", remoteID, c)
 
 		s.model.AddConnection(modelConn, hello)
-		continue next
+		continue
 	}
 }
 
-func (s *service) connect() {
+func (s *service) connect(stop chan struct{}) {
 	nextDial := make(map[string]time.Time)
 
 	// Used as delay for the first few connection attempts, increases
@@ -465,11 +472,16 @@ func (s *service) connect() {
 
 		if initialRampup < sleep {
 			l.Debugln("initial rampup; sleep", initialRampup, "and update to", initialRampup*2)
-			time.Sleep(initialRampup)
+			sleep = initialRampup
 			initialRampup *= 2
 		} else {
 			l.Debugln("sleep until next dial", sleep)
-			time.Sleep(sleep)
+		}
+
+		select {
+		case <-time.After(sleep):
+		case <-stop:
+			return
 		}
 	}
 }
@@ -542,7 +554,7 @@ func (s *service) createListener(factory listenerFactory, uri *url.URL) bool {
 }
 
 func (s *service) logListenAddressesChangedEvent(l genericListener) {
-	events.Default.Log(events.ListenAddressesChanged, map[string]interface{}{
+	s.evLogger.Log(events.ListenAddressesChanged, map[string]interface{}{
 		"address": l.URI(),
 		"lan":     l.LANAddresses(),
 		"wan":     l.WANAddresses(),
@@ -569,7 +581,7 @@ func (s *service) CommitConfiguration(from, to config.Configuration) bool {
 
 	s.listenersMut.Lock()
 	seen := make(map[string]struct{})
-	for _, addr := range config.Wrap("", to).ListenAddresses() {
+	for _, addr := range config.Wrap("", to, s.evLogger).ListenAddresses() {
 		if addr == "" {
 			// We can get an empty address if there is an empty listener
 			// element in the config, indicating no listeners should be
@@ -839,8 +851,15 @@ func (s *service) dialParallel(deviceID protocol.DeviceID, dialTargets []dialTar
 			wg.Add(1)
 			go func(tgt dialTarget) {
 				conn, err := tgt.Dial()
-				s.setConnectionStatus(tgt.addr, err)
 				if err == nil {
+					// Closes the connection on error
+					err = s.validateIdentity(conn, deviceID)
+				}
+				s.setConnectionStatus(tgt.addr, err)
+				if err != nil {
+					l.Debugln("dialing", deviceID, tgt.uri, "error:", err)
+				} else {
+					l.Debugln("dialing", deviceID, tgt.uri, "success:", conn)
 					res <- conn
 				}
 				wg.Done()
@@ -873,3 +892,36 @@ func (s *service) dialParallel(deviceID protocol.DeviceID, dialTargets []dialTar
 	}
 	return internalConn{}, false
 }
+
+func (s *service) validateIdentity(c internalConn, expectedID protocol.DeviceID) error {
+	cs := c.ConnectionState()
+
+	// We should have received exactly one certificate from the other
+	// side. If we didn't, they don't have a device ID and we drop the
+	// connection.
+	certs := cs.PeerCertificates
+	if cl := len(certs); cl != 1 {
+		l.Infof("Got peer certificate list of length %d != 1 from peer at %s; protocol error", cl, c)
+		c.Close()
+		return fmt.Errorf("expected 1 certificate, got %d", cl)
+	}
+	remoteCert := certs[0]
+	remoteID := protocol.NewDeviceID(remoteCert.Raw)
+
+	// The device ID should not be that of ourselves. It can happen
+	// though, especially in the presence of NAT hairpinning, multiple
+	// clients between the same NAT gateway, and global discovery.
+	if remoteID == s.myID {
+		l.Infof("Connected to myself (%s) at %s - should not happen", remoteID, c)
+		c.Close()
+		return fmt.Errorf("connected to self")
+	}
+
+	// We should see the expected device ID
+	if !remoteID.Equals(expectedID) {
+		c.Close()
+		return fmt.Errorf("unexpected device id, expected %s got %s", expectedID, remoteID)
+	}
+
+	return nil
+}

lib/connections/structs.go

@@ -191,13 +191,6 @@ type Model interface {
 	GetHello(protocol.DeviceID) protocol.HelloIntf
 }
 
-// serviceFunc wraps a function to create a suture.Service without stop
-// functionality.
-type serviceFunc func()
-
-func (f serviceFunc) Serve() { f() }
-func (f serviceFunc) Stop()  {}
-
 type onAddressesChangedNotifier struct {
 	callbacks []func(genericListener)
 }
@@ -222,11 +215,5 @@ type dialTarget struct {
 
 func (t dialTarget) Dial() (internalConn, error) {
 	l.Debugln("dialing", t.deviceID, t.uri, "prio", t.priority)
-	conn, err := t.dialer.Dial(t.deviceID, t.uri)
-	if err != nil {
-		l.Debugln("dialing", t.deviceID, t.uri, "error:", err)
-	} else {
-		l.Debugln("dialing", t.deviceID, t.uri, "success:", conn)
-	}
-	return conn, err
+	return t.dialer.Dial(t.deviceID, t.uri)
 }

lib/connections/tcp_dial.go

@@ -30,7 +30,7 @@ type tcpDialer struct {
 	tlsCfg *tls.Config
 }
 
-func (d *tcpDialer) Dial(id protocol.DeviceID, uri *url.URL) (internalConn, error) {
+func (d *tcpDialer) Dial(_ protocol.DeviceID, uri *url.URL) (internalConn, error) {
 	uri = fixupPort(uri, config.DefaultTCPPort)
 
 	conn, err := dialer.DialTimeout(uri.Scheme, uri.Host, 10*time.Second)

lib/connections/tcp_listen.go

@@ -16,6 +16,7 @@ import (
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/dialer"
 	"github.com/syncthing/syncthing/lib/nat"
+	"github.com/syncthing/syncthing/lib/util"
 )
 
 func init() {
@@ -26,43 +27,32 @@ func init() {
 }
 
 type tcpListener struct {
+	util.ServiceWithError
 	onAddressesChangedNotifier
 
 	uri     *url.URL
 	cfg     config.Wrapper
 	tlsCfg  *tls.Config
-	stop    chan struct{}
 	conns   chan internalConn
 	factory listenerFactory
 
 	natService *nat.Service
 	mapping    *nat.Mapping
 
-	err error
 	mut sync.RWMutex
 }
 
-func (t *tcpListener) Serve() {
-	t.mut.Lock()
-	t.err = nil
-	t.mut.Unlock()
-
+func (t *tcpListener) serve(stop chan struct{}) error {
 	tcaddr, err := net.ResolveTCPAddr(t.uri.Scheme, t.uri.Host)
 	if err != nil {
-		t.mut.Lock()
-		t.err = err
-		t.mut.Unlock()
 		l.Infoln("Listen (BEP/tcp):", err)
-		return
+		return err
 	}
 
 	listener, err := net.ListenTCP(t.uri.Scheme, tcaddr)
 	if err != nil {
-		t.mut.Lock()
-		t.err = err
-		t.mut.Unlock()
 		l.Infoln("Listen (BEP/tcp):", err)
-		return
+		return err
 	}
 	defer listener.Close()
 
@@ -86,14 +76,14 @@ func (t *tcpListener) Serve() {
 		listener.SetDeadline(time.Now().Add(time.Second))
 		conn, err := listener.Accept()
 		select {
-		case <-t.stop:
+		case <-stop:
 			if err == nil {
 				conn.Close()
 			}
 			t.mut.Lock()
 			t.mapping = nil
 			t.mut.Unlock()
-			return
+			return nil
 		default:
 		}
 		if err != nil {
@@ -104,7 +94,7 @@ func (t *tcpListener) Serve() {
 				if acceptFailures > maxAcceptFailures {
 					// Return to restart the listener, because something
 					// seems permanently damaged.
-					return
+					return err
 				}
 
 				// Slightly increased delay for each failure.
@@ -137,10 +127,6 @@ func (t *tcpListener) Serve() {
 	}
 }
 
-func (t *tcpListener) Stop() {
-	close(t.stop)
-}
-
 func (t *tcpListener) URI() *url.URL {
 	return t.uri
 }
@@ -174,13 +160,6 @@ func (t *tcpListener) LANAddresses() []*url.URL {
 	return []*url.URL{t.uri}
 }
 
-func (t *tcpListener) Error() error {
-	t.mut.RLock()
-	err := t.err
-	t.mut.RUnlock()
-	return err
-}
-
 func (t *tcpListener) String() string {
 	return t.uri.String()
 }
@@ -196,15 +175,16 @@ func (t *tcpListener) NATType() string {
 type tcpListenerFactory struct{}
 
 func (f *tcpListenerFactory) New(uri *url.URL, cfg config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {
-	return &tcpListener{
+	l := &tcpListener{
 		uri:        fixupPort(uri, config.DefaultTCPPort),
 		cfg:        cfg,
 		tlsCfg:     tlsCfg,
 		conns:      conns,
 		natService: natService,
-		stop:       make(chan struct{}),
 		factory:    f,
 	}
+	l.ServiceWithError = util.AsServiceWithError(l.serve)
+	return l
 }
 
 func (tcpListenerFactory) Valid(_ config.Configuration) error {

lib/db/benchmark_test.go

@@ -16,13 +16,13 @@ import (
 )
 
 var files, oneFile, firstHalf, secondHalf []protocol.FileInfo
-var benchS *db.FileSet
 
-func lazyInitBenchFileSet() {
-	if benchS != nil {
+func lazyInitBenchFiles() {
+	if files != nil {
 		return
 	}
 
+	files = make([]protocol.FileInfo, 0, 1000)
 	for i := 0; i < 1000; i++ {
 		files = append(files, protocol.FileInfo{
 			Name:    fmt.Sprintf("file%d", i),
@@ -35,11 +35,17 @@ func lazyInitBenchFileSet() {
 	firstHalf = files[:middle]
 	secondHalf = files[middle:]
 	oneFile = firstHalf[middle-1 : middle]
+}
+
+func getBenchFileSet() (*db.Lowlevel, *db.FileSet) {
+	lazyInitBenchFiles()
 
 	ldb := db.OpenMemory()
-	benchS = db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
+	benchS := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
 	replace(benchS, remoteDevice0, files)
 	replace(benchS, protocol.LocalDeviceID, firstHalf)
+
+	return ldb, benchS
 }
 
 func BenchmarkReplaceAll(b *testing.B) {
@@ -56,7 +62,8 @@ func BenchmarkReplaceAll(b *testing.B) {
 }
 
 func BenchmarkUpdateOneChanged(b *testing.B) {
-	lazyInitBenchFileSet()
+	ldb, benchS := getBenchFileSet()
+	defer ldb.Close()
 
 	changed := make([]protocol.FileInfo, 1)
 	changed[0] = oneFile[0]
@@ -75,7 +82,8 @@ func BenchmarkUpdateOneChanged(b *testing.B) {
 }
 
 func BenchmarkUpdate100Changed(b *testing.B) {
-	lazyInitBenchFileSet()
+	ldb, benchS := getBenchFileSet()
+	defer ldb.Close()
 
 	unchanged := files[100:200]
 	changed := append([]protocol.FileInfo{}, unchanged...)
@@ -96,7 +104,8 @@ func BenchmarkUpdate100Changed(b *testing.B) {
 }
 
 func BenchmarkUpdate100ChangedRemote(b *testing.B) {
-	lazyInitBenchFileSet()
+	ldb, benchS := getBenchFileSet()
+	defer ldb.Close()
 
 	unchanged := files[100:200]
 	changed := append([]protocol.FileInfo{}, unchanged...)
@@ -117,7 +126,8 @@ func BenchmarkUpdate100ChangedRemote(b *testing.B) {
 }
 
 func BenchmarkUpdateOneUnchanged(b *testing.B) {
-	lazyInitBenchFileSet()
+	ldb, benchS := getBenchFileSet()
+	defer ldb.Close()
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -128,7 +138,8 @@ func BenchmarkUpdateOneUnchanged(b *testing.B) {
 }
 
 func BenchmarkNeedHalf(b *testing.B) {
-	lazyInitBenchFileSet()
+	ldb, benchS := getBenchFileSet()
+	defer ldb.Close()
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -146,8 +157,6 @@ func BenchmarkNeedHalf(b *testing.B) {
 }
 
 func BenchmarkNeedHalfRemote(b *testing.B) {
-	lazyInitBenchFileSet()
-
 	ldb := db.OpenMemory()
 	defer ldb.Close()
 	fset := db.NewFileSet("test)", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
@@ -170,7 +179,8 @@ func BenchmarkNeedHalfRemote(b *testing.B) {
 }
 
 func BenchmarkHave(b *testing.B) {
-	lazyInitBenchFileSet()
+	ldb, benchS := getBenchFileSet()
+	defer ldb.Close()
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -188,7 +198,8 @@ func BenchmarkHave(b *testing.B) {
 }
 
 func BenchmarkGlobal(b *testing.B) {
-	lazyInitBenchFileSet()
+	ldb, benchS := getBenchFileSet()
+	defer ldb.Close()
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -206,7 +217,8 @@ func BenchmarkGlobal(b *testing.B) {
 }
 
 func BenchmarkNeedHalfTruncated(b *testing.B) {
-	lazyInitBenchFileSet()
+	ldb, benchS := getBenchFileSet()
+	defer ldb.Close()
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -224,7 +236,8 @@ func BenchmarkNeedHalfTruncated(b *testing.B) {
 }
 
 func BenchmarkHaveTruncated(b *testing.B) {
-	lazyInitBenchFileSet()
+	ldb, benchS := getBenchFileSet()
+	defer ldb.Close()
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -242,7 +255,8 @@ func BenchmarkHaveTruncated(b *testing.B) {
 }
 
 func BenchmarkGlobalTruncated(b *testing.B) {
-	lazyInitBenchFileSet()
+	ldb, benchS := getBenchFileSet()
+	defer ldb.Close()
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {

lib/db/db_test.go

@@ -167,7 +167,7 @@ func TestUpdate0to3(t *testing.T) {
 			t.Error("Unexpected additional file via sequence", f.FileName())
 			return true
 		}
-		if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalentOptional(e, true, true, 0) {
+		if e := haveUpdate0to3[protocol.LocalDeviceID][0]; f.IsEquivalentOptional(e, 0, true, true, 0) {
 			found = true
 		} else {
 			t.Errorf("Wrong file via sequence, got %v, expected %v", f, e)
@@ -192,7 +192,7 @@ func TestUpdate0to3(t *testing.T) {
 		}
 		f := fi.(protocol.FileInfo)
 		delete(need, f.Name)
-		if !f.IsEquivalentOptional(e, true, true, 0) {
+		if !f.IsEquivalentOptional(e, 0, true, true, 0) {
 			t.Errorf("Wrong needed file, got %v, expected %v", f, e)
 		}
 		return true

lib/db/debug.go

@@ -7,9 +7,6 @@
 package db
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
@@ -17,10 +14,6 @@ var (
 	l = logger.DefaultLogger.NewFacility("db", "The database layer")
 )
 
-func init() {
-	l.SetDebug("db", strings.Contains(os.Getenv("STTRACE"), "db") || os.Getenv("STTRACE") == "all")
-}
-
 func shouldDebug() bool {
 	return l.ShouldDebug("db")
 }

lib/db/instance.go

@@ -172,7 +172,8 @@ func (db *instance) withHaveSequence(folder []byte, startSeq int64, fn Iterator)
 
 		if shouldDebug() {
 			if seq := db.keyer.SequenceFromSequenceKey(dbi.Key()); f.Sequence != seq {
-				panic(fmt.Sprintf("sequence index corruption (folder %v, file %v): sequence %d != expected %d", string(folder), f.Name, f.Sequence, seq))
+				l.Warnf("Sequence index corruption (folder %v, file %v): sequence %d != expected %d", string(folder), f.Name, f.Sequence, seq)
+				panic("sequence index corruption")
 			}
 		}
 		if !fn(f) {
@@ -561,7 +562,7 @@ func (e errorSuggestion) Error() string {
 
 // unchanged checks if two files are the same and thus don't need to be updated.
 // Local flags or the invalid bit might change without the version
-// being bumped. The IsInvalid() method handles both.
+// being bumped.
 func unchanged(nf, ef FileIntf) bool {
-	return ef.FileVersion().Equal(nf.FileVersion()) && ef.IsInvalid() == nf.IsInvalid()
+	return ef.FileVersion().Equal(nf.FileVersion()) && ef.IsInvalid() == nf.IsInvalid() && ef.FileLocalFlags() == nf.FileLocalFlags()
 }

lib/db/lowlevel.go

@@ -8,6 +8,7 @@ package db
 
 import (
 	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -22,8 +23,26 @@ import (
 
 const (
 	dbMaxOpenFiles = 100
-	dbWriteBuffer  = 16 << 20
-	dbFlushBatch   = dbWriteBuffer / 4 // Some leeway for any leveldb in-memory optimizations
+	dbFlushBatch   = 4 << MiB
+
+	// A large database is > 200 MiB. It's a mostly arbitrary value, but
+	// it's also the case that each file is 2 MiB by default and when we
+	// have dbMaxOpenFiles of them we will need to start thrashing fd:s.
+	// Switching to large database settings causes larger files to be used
+	// when compacting, reducing the number.
+	dbLargeThreshold = dbMaxOpenFiles * (2 << MiB)
+
+	KiB = 10
+	MiB = 20
+)
+
+type Tuning int
+
+const (
+	// N.b. these constants must match those in lib/config.Tuning!
+	TuningAuto Tuning = iota
+	TuningSmall
+	TuningLarge
 )
 
 // Lowlevel is the lowest level database interface. It has a very simple
@@ -45,14 +64,79 @@ type Lowlevel struct {
 // Open attempts to open the database at the given location, and runs
 // recovery on it if opening fails. Worst case, if recovery is not possible,
 // the database is erased and created from scratch.
-func Open(location string) (*Lowlevel, error) {
-	opts := &opt.Options{
-		OpenFilesCacheCapacity: dbMaxOpenFiles,
-		WriteBuffer:            dbWriteBuffer,
-	}
+func Open(location string, tuning Tuning) (*Lowlevel, error) {
+	opts := optsFor(location, tuning)
 	return open(location, opts)
 }
 
+// optsFor returns the database options to use when opening a database with
+// the given location and tuning. Settings can be overridden by debug
+// environment variables.
+func optsFor(location string, tuning Tuning) *opt.Options {
+	large := false
+	switch tuning {
+	case TuningLarge:
+		large = true
+	case TuningAuto:
+		large = dbIsLarge(location)
+	}
+
+	var (
+		// Set defaults used for small databases.
+		defaultBlockCacheCapacity            = 0 // 0 means let leveldb use default
+		defaultBlockSize                     = 0
+		defaultCompactionTableSize           = 0
+		defaultCompactionTableSizeMultiplier = 0
+		defaultWriteBuffer                   = 16 << MiB                      // increased from leveldb default of 4 MiB
+		defaultCompactionL0Trigger           = opt.DefaultCompactionL0Trigger // explicit because we use it as base for other stuff
+	)
+
+	if large {
+		// Change the parameters for better throughput at the price of some
+		// RAM and larger files. This results in larger batches of writes
+		// and compaction at a lower frequency.
+		l.Infoln("Using large-database tuning")
+
+		defaultBlockCacheCapacity = 64 << MiB
+		defaultBlockSize = 64 << KiB
+		defaultCompactionTableSize = 16 << MiB
+		defaultCompactionTableSizeMultiplier = 20 // 2.0 after division by ten
+		defaultWriteBuffer = 64 << MiB
+		defaultCompactionL0Trigger = 8 // number of l0 files
+	}
+
+	opts := &opt.Options{
+		BlockCacheCapacity:            debugEnvValue("BlockCacheCapacity", defaultBlockCacheCapacity),
+		BlockCacheEvictRemoved:        debugEnvValue("BlockCacheEvictRemoved", 0) != 0,
+		BlockRestartInterval:          debugEnvValue("BlockRestartInterval", 0),
+		BlockSize:                     debugEnvValue("BlockSize", defaultBlockSize),
+		CompactionExpandLimitFactor:   debugEnvValue("CompactionExpandLimitFactor", 0),
+		CompactionGPOverlapsFactor:    debugEnvValue("CompactionGPOverlapsFactor", 0),
+		CompactionL0Trigger:           debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger),
+		CompactionSourceLimitFactor:   debugEnvValue("CompactionSourceLimitFactor", 0),
+		CompactionTableSize:           debugEnvValue("CompactionTableSize", defaultCompactionTableSize),
+		CompactionTableSizeMultiplier: float64(debugEnvValue("CompactionTableSizeMultiplier", defaultCompactionTableSizeMultiplier)) / 10.0,
+		CompactionTotalSize:           debugEnvValue("CompactionTotalSize", 0),
+		CompactionTotalSizeMultiplier: float64(debugEnvValue("CompactionTotalSizeMultiplier", 0)) / 10.0,
+		DisableBufferPool:             debugEnvValue("DisableBufferPool", 0) != 0,
+		DisableBlockCache:             debugEnvValue("DisableBlockCache", 0) != 0,
+		DisableCompactionBackoff:      debugEnvValue("DisableCompactionBackoff", 0) != 0,
+		DisableLargeBatchTransaction:  debugEnvValue("DisableLargeBatchTransaction", 0) != 0,
+		NoSync:                        debugEnvValue("NoSync", 0) != 0,
+		NoWriteMerge:                  debugEnvValue("NoWriteMerge", 0) != 0,
+		OpenFilesCacheCapacity:        debugEnvValue("OpenFilesCacheCapacity", dbMaxOpenFiles),
+		WriteBuffer:                   debugEnvValue("WriteBuffer", defaultWriteBuffer),
+		// The write slowdown and pause can be overridden, but even if they
+		// are not and the compaction trigger is overridden we need to
+		// adjust so that we don't pause writes for L0 compaction before we
+		// even *start* L0 compaction...
+		WriteL0SlowdownTrigger: debugEnvValue("WriteL0SlowdownTrigger", 2*debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger)),
+		WriteL0PauseTrigger:    debugEnvValue("WriteL0SlowdownTrigger", 3*debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger)),
+	}
+
+	return opts
+}
+
 // OpenRO attempts to open the database at the given location, read only.
 func OpenRO(location string) (*Lowlevel, error) {
 	opts := &opt.Options{
@@ -80,6 +164,13 @@ func open(location string, opts *opt.Options) (*Lowlevel, error) {
 	if err != nil {
 		return nil, errorSuggestion{err, "is another instance of Syncthing running?"}
 	}
+
+	if debugEnvValue("CompactEverything", 0) != 0 {
+		if err := db.CompactRange(util.Range{}); err != nil {
+			l.Warnln("Compacting database:", err)
+		}
+	}
+
 	return NewLowlevel(db, location), nil
 }
 
@@ -173,6 +264,38 @@ func (db *Lowlevel) Close() {
 	db.DB.Close()
 }
 
+// dbIsLarge returns whether the estimated size of the database at location
+// is large enough to warrant optimization for large databases.
+func dbIsLarge(location string) bool {
+	if ^uint(0)>>63 == 0 {
+		// We're compiled for a 32 bit architecture. We've seen trouble with
+		// large settings there.
+		// (https://forum.syncthing.net/t/many-small-ldb-files-with-database-tuning/13842)
+		return false
+	}
+
+	dir, err := os.Open(location)
+	if err != nil {
+		return false
+	}
+
+	fis, err := dir.Readdir(-1)
+	if err != nil {
+		return false
+	}
+
+	var size int64
+	for _, fi := range fis {
+		if fi.Name() == "LOG" {
+			// don't count the size
+			continue
+		}
+		size += fi.Size()
+	}
+
+	return size > dbLargeThreshold
+}
+
 // NewLowlevel wraps the given *leveldb.DB into a *lowlevel
 func NewLowlevel(db *leveldb.DB, location string) *Lowlevel {
 	return &Lowlevel{
@@ -304,3 +427,11 @@ func (it *iter) execIfNotClosed(fn func() bool) bool {
 	}
 	return fn()
 }
+
+func debugEnvValue(key string, def int) int {
+	v, err := strconv.ParseInt(os.Getenv("STDEBUG_"+key), 10, 63)
+	if err != nil {
+		return def
+	}
+	return int(v)
+}

lib/db/set.go

@@ -51,6 +51,10 @@ type FileIntf interface {
 	SequenceNo() int64
 	BlockSize() int
 	FileVersion() protocol.Vector
+	FileType() protocol.FileInfoType
+	FilePermissions() uint32
+	FileModifiedBy() protocol.ShortID
+	ModTime() time.Time
 }
 
 // The Iterator is called with either a protocol.FileInfo or a

lib/db/set_test.go

@@ -729,7 +729,7 @@ func BenchmarkUpdateOneFile(b *testing.B) {
 		protocol.FileInfo{Name: "zajksdhaskjdh/askjdhaskjdashkajshd/kasjdhaskjdhaskdjhaskdjash/dkjashdaksjdhaskdjahskdjh", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
 	}
 
-	ldb, err := db.Open("testdata/benchmarkupdate.db")
+	ldb, err := db.Open("testdata/benchmarkupdate.db", db.TuningAuto)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -905,7 +905,7 @@ func TestWithHaveSequence(t *testing.T) {
 
 	i := 2
 	s.WithHaveSequence(int64(i), func(fi db.FileIntf) bool {
-		if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1]) {
+		if f := fi.(protocol.FileInfo); !f.IsEquivalent(localHave[i-1], 0) {
 			t.Fatalf("Got %v\nExpected %v", f, localHave[i-1])
 		}
 		i++
@@ -1004,7 +1004,7 @@ func TestMoveGlobalBack(t *testing.T) {
 
 	if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
 		t.Error("Expected 1 local need, got", need)
-	} else if !need[0].IsEquivalent(remote0Have[0]) {
+	} else if !need[0].IsEquivalent(remote0Have[0], 0) {
 		t.Errorf("Local need incorrect;\n A: %v !=\n E: %v", need[0], remote0Have[0])
 	}
 
@@ -1030,7 +1030,7 @@ func TestMoveGlobalBack(t *testing.T) {
 
 	if need := needList(s, remoteDevice0); len(need) != 1 {
 		t.Error("Expected 1 need for remote 0, got", need)
-	} else if !need[0].IsEquivalent(localHave[0]) {
+	} else if !need[0].IsEquivalent(localHave[0], 0) {
 		t.Errorf("Need for remote 0 incorrect;\n A: %v !=\n E: %v", need[0], localHave[0])
 	}
 
@@ -1066,7 +1066,7 @@ func TestIssue5007(t *testing.T) {
 
 	if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
 		t.Fatal("Expected 1 local need, got", need)
-	} else if !need[0].IsEquivalent(fs[0]) {
+	} else if !need[0].IsEquivalent(fs[0], 0) {
 		t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0])
 	}
 
@@ -1101,7 +1101,7 @@ func TestNeedDeleted(t *testing.T) {
 
 	if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
 		t.Fatal("Expected 1 local need, got", need)
-	} else if !need[0].IsEquivalent(fs[0]) {
+	} else if !need[0].IsEquivalent(fs[0], 0) {
 		t.Fatalf("Local need incorrect;\n A: %v !=\n E: %v", need[0], fs[0])
 	}
 
@@ -1243,7 +1243,7 @@ func TestNeedAfterUnignore(t *testing.T) {
 
 	if need := needList(s, protocol.LocalDeviceID); len(need) != 1 {
 		t.Fatal("Expected one local need, got", need)
-	} else if !need[0].IsEquivalent(remote) {
+	} else if !need[0].IsEquivalent(remote, 0) {
 		t.Fatalf("Got %v, expected %v", need[0], remote)
 	}
 }
@@ -1287,7 +1287,7 @@ func TestNeedWithNewerInvalid(t *testing.T) {
 	if len(need) != 1 {
 		t.Fatal("Locally missing file should be needed")
 	}
-	if !need[0].IsEquivalent(file) {
+	if !need[0].IsEquivalent(file, 0) {
 		t.Fatalf("Got needed file %v, expected %v", need[0], file)
 	}
 
@@ -1302,7 +1302,7 @@ func TestNeedWithNewerInvalid(t *testing.T) {
 	if len(need) != 1 {
 		t.Fatal("Locally missing file should be needed regardless of invalid files")
 	}
-	if !need[0].IsEquivalent(file) {
+	if !need[0].IsEquivalent(file, 0) {
 		t.Fatalf("Got needed file %v, expected %v", need[0], file)
 	}
 }
@@ -1462,6 +1462,33 @@ func TestSequenceIndex(t *testing.T) {
 	}
 }
 
+func TestIgnoreAfterReceiveOnly(t *testing.T) {
+	ldb := db.OpenMemory()
+
+	file := "foo"
+	s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
+
+	fs := fileList{{
+		Name:       file,
+		Version:    protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}},
+		LocalFlags: protocol.FlagLocalReceiveOnly,
+	}}
+
+	s.Update(protocol.LocalDeviceID, fs)
+
+	fs[0].LocalFlags = protocol.FlagLocalIgnored
+
+	s.Update(protocol.LocalDeviceID, fs)
+
+	if f, ok := s.Get(protocol.LocalDeviceID, file); !ok {
+		t.Error("File missing in db")
+	} else if f.IsReceiveOnlyChanged() {
+		t.Error("File is still receive-only changed")
+	} else if !f.IsIgnored() {
+		t.Error("File is not ignored")
+	}
+}
+
 func replace(fs *db.FileSet, device protocol.DeviceID, files []protocol.FileInfo) {
 	fs.Drop(device)
 	fs.Update(device, files)

lib/db/structs.go

@@ -116,6 +116,17 @@ func (f FileInfoTruncated) FileVersion() protocol.Vector {
 	return f.Version
 }
 
+func (f FileInfoTruncated) FileType() protocol.FileInfoType {
+	return f.Type
+}
+
+func (f FileInfoTruncated) FilePermissions() uint32 {
+	return f.Permissions
+}
+
+func (f FileInfoTruncated) FileModifiedBy() protocol.ShortID {
+	return f.ModifiedBy
+}
 func (f FileInfoTruncated) ConvertToIgnoredFileInfo(by protocol.ShortID) protocol.FileInfo {
 	return protocol.FileInfo{
 		Name:         f.Name,

lib/dialer/internal.go

@@ -11,7 +11,6 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"strings"
 	"time"
 
 	"golang.org/x/net/proxy"
@@ -29,8 +28,6 @@ var (
 type dialFunc func(network, addr string) (net.Conn, error)
 
 func init() {
-	l.SetDebug("dialer", strings.Contains(os.Getenv("STTRACE"), "dialer") || os.Getenv("STTRACE") == "all")
-
 	proxy.RegisterDialerType("socks", socksDialerFunction)
 	proxyDialer = getDialer(proxy.Direct)
 	usingProxy = proxyDialer != proxy.Direct