lib/db, lib/protocol: Compact FileInfo and BlockInfo alignment (#6215)

* lib/db, lib/protocol: Compact FileInfo and BlockInfo alignment

This fixes the following two lint warnings

    FileInfo: struct of size 160 bytes could be of size 136 bytes
    BlockInfo: struct of size 48 bytes could be of size 40 bytes

by reordering fields in alignment order (64 bit fields, then 32 bit
fields, then 16 bit fields (if any), then small ones). The end result is
a slightly less aesthetically pleasing struct field order, but since
these are the objects we often juggle in bulk and keep large queues of I
think it's worth it.

It's a micro optimization, but a cheap one.

.github/ISSUE_TEMPLATE.md

@@ -36,7 +36,7 @@ its entirety.
 
 ### Version Information
 
-Syncthing Version: v0.x.y
+Syncthing Version: v1.x.y
 OS Version: Windows 7 / Ubuntu 14.04 / ...
 Browser Version: (if applicable, for GUI issues)
 

.golangci.yml

@@ -12,9 +12,14 @@ linters:
     - gochecknoglobals
     - gofmt
     - scopelint
+    - gocyclo
+    - funlen
+    - wsl
+    - gocognit
 
 service:
-  golangci-lint-version: 1.16.x
+  golangci-lint-version: 1.21.x
   prepare:
+    - rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
     - GO111MODULE=on go mod vendor
     - go run build.go assets

AUTHORS

@@ -46,7 +46,7 @@ Brandon Philips (philips) <brandon@ifup.org>
 Brendan Long (brendanlong) <self@brendanlong.com>
 Brian R. Becker (brbecker) <brbecker@gmail.com>
 Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
-Carsten Hagemann (Moter8) <moter8@gmail.com>
+Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
 Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
 Cedric Staniewski (xduugu) <cedric@gmx.ca>
 Chris Howie (cdhowie) <me@chrishowie.com>

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.12 AS builder
+FROM golang:1.13 AS builder
 
 WORKDIR /src
 COPY . .
@@ -19,9 +19,10 @@ RUN apk add --no-cache ca-certificates su-exec
 COPY --from=builder /src/syncthing /bin/syncthing
 COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
 
-ENV PUID=1000 PGID=1000
+ENV PUID=1000 PGID=1000 HOME=/var/syncthing
 
 HEALTHCHECK --interval=1m --timeout=10s \
   CMD nc -z localhost 8384 || exit 1
 
-ENTRYPOINT ["/bin/entrypoint.sh", "-home", "/var/syncthing/config", "-gui-address", "0.0.0.0:8384"]
+ENV STGUIADDRESS=0.0.0.0:8384
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]

Dockerfile.stdiscosrv

@@ -0,0 +1,28 @@
+FROM golang:1.13 AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f stdiscosrv && go run build.go -no-upgrade build stdiscosrv
+
+FROM alpine
+
+EXPOSE 19200 8443
+
+VOLUME ["/var/stdiscosrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/stdiscosrv /bin/stdiscosrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/stdiscosrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 8443 || exit 1
+
+WORKDIR /var/stdiscosrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/stdiscosrv"]

Dockerfile.strelaysrv

@@ -0,0 +1,28 @@
+FROM golang:1.13 AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f strelaysrv && go run build.go -no-upgrade build strelaysrv
+
+FROM alpine
+
+EXPOSE 22067 22070
+
+VOLUME ["/var/strelaysrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/strelaysrv /bin/strelaysrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/strelaysrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 22067 || exit 1
+
+WORKDIR /var/strelaysrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaysrv"]

README-Docker.md

@@ -18,7 +18,11 @@ $ docker run -p 8384:8384 -p 22000:22000 \
     syncthing/syncthing:latest
 ```
 
-Note that local device discovery will not work with the above command, resulting in poor local transfer rates if local device addresses are not manually configured.
+## Discovery
+
+Note that local device discovery will not work with the above command,
+resulting in poor local transfer rates if local device addresses are not
+manually configured.
 
 To allow local discovery, the docker host network can be used instead:
 
@@ -32,3 +36,24 @@ $ docker run --network=host \
 Be aware that syncthing alone is now in control of what interfaces and ports it
 listens on. You can edit the syncthing configuration to change the defaults if
 there are conflicts.
+
+## GUI Security
+
+By default Syncthing inside the Docker image listens on 0.0.0.0:8384 to
+allow GUI connections via the Docker proxy. This is set by the
+`STGUIADDRESS` environment variable in the Dockerfile, as it differs from
+what Syncthing would otherwise use by default. This means you should set up
+authentication in the GUI, like for any other externally reachable Syncthing
+instance. If you do not require the GUI, or you use host networking, you can
+unset the `STGUIADDRESS` variable to have Syncthing fall back to listening
+on 127.0.0.1:
+
+```
+$ docker pull syncthing/syncthing
+$ docker run -e STGUIADDRESS= \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+With the environment variable unset Syncthing will follow what is set in the
+configuration file / GUI settings dialog.

README.md

@@ -62,6 +62,10 @@ There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory][3]. There are also several [GUI
 implementations][11] for Windows, Mac and Linux.
 
+## Docker
+
+To run Syncthing in Docker, see [the Docker README][16].
+
 ## Vote on features/bugs
 
 We'd like to encourage you to [vote][12] on issues that matter to you.
@@ -110,4 +114,5 @@ All code is licensed under the [MPLv2 License][7].
 [13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
 [14]: assets/logo-text-128.png
 [15]: https://syncthing.net/
+[16]: https://github.com/syncthing/syncthing/blob/master/README-Docker.md
 

build.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"os/exec"
 	"os/user"
+	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -34,34 +35,35 @@ import (
 )
 
 var (
-	versionRe        = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
-	goarch           string
-	goos             string
-	noupgrade        bool
-	version          string
-	goCmd            string
-	goVersion        float64
-	race             bool
-	debug            = os.Getenv("BUILDDEBUG") != ""
-	extraTags        string
-	installSuffix    string
-	pkgdir           string
-	cc               string
-	debugBinary      bool
-	coverage         bool
-	timeout          = "120s"
-	gogoProtoVersion = "v1.2.0"
+	versionRe     = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
+	goarch        string
+	goos          string
+	noupgrade     bool
+	version       string
+	goCmd         string
+	goVersion     float64
+	race          bool
+	debug         = os.Getenv("BUILDDEBUG") != ""
+	extraTags     string
+	installSuffix string
+	pkgdir        string
+	cc            string
+	debugBinary   bool
+	coverage      bool
+	timeout       = "120s"
 )
 
 type target struct {
 	name              string
 	debname           string
 	debdeps           []string
+	debpre            string
 	debpost           string
 	description       string
-	buildPkg          string
+	buildPkgs         []string
 	binaryName        string
 	archiveFiles      []archiveFile
+	systemdServices   []string
 	installationFiles []archiveFile
 	tags              []string
 }
@@ -75,9 +77,8 @@ type archiveFile struct {
 var targets = map[string]target{
 	"all": {
 		// Only valid for the "build" and "install" commands as it lacks all
-		// the archive creation stuff.
-		buildPkg: "github.com/syncthing/syncthing/cmd/...",
-		tags:     []string{"purego"},
+		// the archive creation stuff. buildPkgs gets filled out in init()
+		tags: []string{"purego"},
 	},
 	"syncthing": {
 		// The default target for "build", "install", "tar", "zip", "deb", etc.
@@ -86,7 +87,7 @@ var targets = map[string]target{
 		debdeps:     []string{"libc6", "procps"},
 		debpost:     "script/post-upgrade",
 		description: "Open Source Continuous File Synchronization",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/syncthing",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/syncthing"},
 		binaryName:  "syncthing", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -128,8 +129,9 @@ var targets = map[string]target{
 		name:        "stdiscosrv",
 		debname:     "syncthing-discosrv",
 		debdeps:     []string{"libc6"},
+		debpre:      "cmd/stdiscosrv/scripts/preinst",
 		description: "Syncthing Discovery Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/stdiscosrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/stdiscosrv"},
 		binaryName:  "stdiscosrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -137,12 +139,17 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
+		systemdServices: []string{
+			"cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service",
+		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
 			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0644},
 			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0644},
 			{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-discosrv", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv", dst: "deb/etc/ufw/applications.d/stdiscosrv", perm: 0644},
 		},
 		tags: []string{"purego"},
 	},
@@ -150,8 +157,9 @@ var targets = map[string]target{
 		name:        "strelaysrv",
 		debname:     "syncthing-relaysrv",
 		debdeps:     []string{"libc6"},
+		debpre:      "cmd/strelaysrv/scripts/preinst",
 		description: "Syncthing Relay Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/strelaysrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaysrv"},
 		binaryName:  "strelaysrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -160,6 +168,9 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
+		systemdServices: []string{
+			"cmd/strelaysrv/etc/linux-systemd/strelaysrv.service",
+		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
 			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0644},
@@ -167,6 +178,8 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0644},
 			{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0644},
+			{src: "cmd/strelaysrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-relaysrv", perm: 0644},
+			{src: "cmd/strelaysrv/etc/firewall-ufw/strelaysrv", dst: "deb/etc/ufw/applications.d/strelaysrv", perm: 0644},
 		},
 	},
 	"strelaypoolsrv": {
@@ -174,7 +187,7 @@ var targets = map[string]target{
 		debname:     "syncthing-relaypoolsrv",
 		debdeps:     []string{"libc6"},
 		description: "Syncthing Relay Pool Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/strelaypoolsrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaypoolsrv"},
 		binaryName:  "strelaypoolsrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -200,11 +213,22 @@ type dependencyRepo struct {
 }
 
 var dependencyRepos = []dependencyRepo{
-	{path: "protobuf", repo: "https://github.com/gogo/protobuf.git", commit: gogoProtoVersion},
 	{path: "xdr", repo: "https://github.com/calmh/xdr.git", commit: "08e072f9cb16"},
 }
 
 func init() {
+	all := targets["all"]
+	pkgs, _ := filepath.Glob("cmd/*")
+	for _, pkg := range pkgs {
+		pkg = filepath.Base(pkg)
+		if strings.HasPrefix(pkg, ".") {
+			// ignore dotfiles
+			continue
+		}
+		all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/cmd/%s", pkg))
+	}
+	targets["all"] = all
+
 	// The "syncthing" target includes a few more files found in the "etc"
 	// and "extra" dirs.
 	syncthingPkg := targets["syncthing"]
@@ -370,9 +394,6 @@ func install(target target, tags []string) {
 	}
 	os.Setenv("GOBIN", filepath.Join(cwd, "bin"))
 
-	args := []string{"install", "-v"}
-	args = appendParameters(args, tags, target)
-
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
 	os.Setenv("CC", cc)
@@ -388,19 +409,20 @@ func install(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	runPrint(goCmd, args...)
+	for _, pkg := range target.buildPkgs {
+		args := []string{"install", "-v"}
+		args = appendParameters(args, tags, pkg)
+
+		runPrint(goCmd, args...)
+	}
 }
 
 func build(target target, tags []string) {
 	lazyRebuildAssets()
-
 	tags = append(target.tags, tags...)
 
 	rmr(target.BinaryName())
 
-	args := []string{"build", "-v"}
-	args = appendParameters(args, tags, target)
-
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
 	os.Setenv("CC", cc)
@@ -420,10 +442,15 @@ func build(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	runPrint(goCmd, args...)
+	for _, pkg := range target.buildPkgs {
+		args := []string{"build", "-v"}
+		args = appendParameters(args, tags, pkg)
+
+		runPrint(goCmd, args...)
+	}
 }
 
-func appendParameters(args []string, tags []string, target target) []string {
+func appendParameters(args []string, tags []string, pkg string) []string {
 	if pkgdir != "" {
 		args = append(args, "-pkgdir", pkgdir)
 	}
@@ -439,7 +466,7 @@ func appendParameters(args []string, tags []string, target target) []string {
 
 	if !debugBinary {
 		// Regular binaries get version tagged and skip some debug symbols
-		args = append(args, "-ldflags", ldflags())
+		args = append(args, "-ldflags", ldflags(path.Base(pkg)))
 	} else {
 		// -gcflags to disable optimizations and inlining. Skip -ldflags
 		// because `Could not launch program: decoding dwarf section info at
@@ -448,7 +475,7 @@ func appendParameters(args []string, tags []string, target target) []string {
 		args = append(args, "-gcflags", "-N -l")
 	}
 
-	return append(args, target.buildPkg)
+	return append(args, pkg)
 }
 
 func buildTar(target target) {
@@ -462,10 +489,7 @@ func buildTar(target target) {
 	}
 
 	build(target, tags)
-
-	if goos == "darwin" {
-		macosCodesign(target.BinaryName())
-	}
+	codesign(target)
 
 	for i := range target.archiveFiles {
 		target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.BinaryName(), 1)
@@ -488,10 +512,7 @@ func buildZip(target target) {
 	}
 
 	build(target, tags)
-
-	if goos == "windows" {
-		windowsCodesign(target.BinaryName())
-	}
+	codesign(target)
 
 	for i := range target.archiveFiles {
 		target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.BinaryName(), 1)
@@ -555,9 +576,15 @@ func buildDeb(target target) {
 	for _, dep := range target.debdeps {
 		args = append(args, "-d", dep)
 	}
+	for _, service := range target.systemdServices {
+		args = append(args, "--deb-systemd", service)
+	}
 	if target.debpost != "" {
 		args = append(args, "--after-upgrade", target.debpost)
 	}
+	if target.debpre != "" {
+		args = append(args, "--before-install", target.debpre)
+	}
 	runPrint("fpm", args...)
 }
 
@@ -690,6 +717,7 @@ func listFiles(dir string) []string {
 		if err != nil {
 			return err
 		}
+
 		if fi.Mode().IsRegular() {
 			res = append(res, path)
 		}
@@ -736,14 +764,21 @@ func shouldRebuildAssets(target, srcdir string) bool {
 }
 
 func proto() {
-	runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", gogoProtoVersion))
+	pv := protobufVersion()
+	dependencyRepos = append(dependencyRepos,
+		dependencyRepo{path: "protobuf", repo: "https://github.com/gogo/protobuf.git", commit: pv},
+	)
+
+	runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
 	os.MkdirAll("repos", 0755)
 	for _, dep := range dependencyRepos {
 		path := filepath.Join("repos", dep.path)
 		if _, err := os.Stat(path); err != nil {
 			runPrintInDir("repos", "git", "clone", dep.repo, dep.path)
-			runPrintInDir(path, "git", "checkout", dep.commit)
+		} else {
+			runPrintInDir(path, "git", "fetch")
 		}
+		runPrintInDir(path, "git", "checkout", dep.commit)
 	}
 	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/stdiscosrv")
 }
@@ -764,7 +799,7 @@ func transifex() {
 	runPrint(goCmd, "run", "../../../../script/transifexdl.go")
 }
 
-func ldflags() string {
+func ldflags(program string) string {
 	sep := '='
 	if goVersion > 0 && goVersion < 1.5 {
 		sep = ' '
@@ -776,6 +811,10 @@ func ldflags() string {
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp%c%d", sep, buildStamp())
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User%c%s", sep, buildUser())
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Host%c%s", sep, buildHost())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Program%c%s", sep, program)
+	if v := os.Getenv("EXTRA_LDFLAGS"); v != "" {
+		fmt.Fprintf(b, " %s", v)
+	}
 	return b.String()
 }
 
@@ -1134,6 +1173,15 @@ func zipFile(out string, files []archiveFile) {
 	}
 }
 
+func codesign(target target) {
+	switch goos {
+	case "windows":
+		windowsCodesign(target.BinaryName())
+	case "darwin":
+		macosCodesign(target.BinaryName())
+	}
+}
+
 func macosCodesign(file string) {
 	if pass := os.Getenv("CODESIGN_KEYCHAIN_PASS"); pass != "" {
 		bs, err := runError("security", "unlock-keychain", "-p", pass)
@@ -1232,3 +1280,11 @@ func (t target) BinaryName() string {
 	}
 	return t.binaryName
 }
+
+func protobufVersion() string {
+	bs, err := runError(goCmd, "list", "-f", "{{.Version}}", "-m", "github.com/gogo/protobuf")
+	if err != nil {
+		log.Fatal("Getting protobuf version:", err)
+	}
+	return string(bs)
+}

cmd/stbench/main.go

@@ -1,143 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// This doesn't build on Windows due to the Rusage stuff.
-
-// +build !windows
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"runtime"
-	"syscall"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/rc"
-)
-
-var homeDir = "h1"
-var syncthingBin = "./bin/syncthing"
-var test = "scan"
-
-func main() {
-	flag.StringVar(&homeDir, "home", homeDir, "Home directory location")
-	flag.StringVar(&syncthingBin, "bin", syncthingBin, "Binary location")
-	flag.StringVar(&test, "test", test, "Test to run")
-	flag.Parse()
-
-	switch test {
-	case "scan":
-		// scan measures the resource usage required to perform the initial
-		// scan, without cleaning away the database first.
-		testScan()
-	}
-}
-
-// testScan starts a process and reports on the resource usage required to
-// perform the initial scan.
-func testScan() {
-	log.Println("Starting...")
-	p := rc.NewProcess("127.0.0.1:8081")
-	if err := p.Start(syncthingBin, "-home", homeDir, "-no-browser"); err != nil {
-		log.Println(err)
-		return
-	}
-	defer p.Stop()
-
-	wallTime := awaitScanComplete(p)
-
-	report(p, wallTime)
-}
-
-// awaitScanComplete waits for a folder to transition idle->scanning and
-// then scanning->idle and returns the time taken for the scan.
-func awaitScanComplete(p *rc.Process) time.Duration {
-	log.Println("Awaiting scan completion...")
-	var t0, t1 time.Time
-	lastEvent := 0
-loop:
-	for {
-		evs, err := p.Events(lastEvent)
-		if err != nil {
-			continue
-		}
-
-		for _, ev := range evs {
-			if ev.Type == "StateChanged" {
-				data := ev.Data.(map[string]interface{})
-				log.Println(ev)
-
-				if data["to"].(string) == "scanning" {
-					t0 = ev.Time
-					continue
-				}
-
-				if !t0.IsZero() && data["to"].(string) == "idle" {
-					t1 = ev.Time
-					break loop
-				}
-			}
-			lastEvent = ev.ID
-		}
-
-		time.Sleep(250 * time.Millisecond)
-	}
-
-	return t1.Sub(t0)
-}
-
-// report stops the given process and reports on its resource usage in two
-// ways: human readable to stderr, and CSV to stdout.
-func report(p *rc.Process, wallTime time.Duration) {
-	sv, err := p.SystemVersion()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	ss, err := p.SystemStatus()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	proc, err := p.Stop()
-	if err != nil {
-		return
-	}
-
-	rusage, ok := proc.SysUsage().(*syscall.Rusage)
-	if !ok {
-		return
-	}
-
-	log.Println("Version:", sv.Version)
-	log.Println("Alloc:", ss.Alloc/1024, "KiB")
-	log.Println("Sys:", ss.Sys/1024, "KiB")
-	log.Println("Goroutines:", ss.Goroutines)
-	log.Println("Wall time:", wallTime)
-	log.Println("Utime:", time.Duration(rusage.Utime.Nano()))
-	log.Println("Stime:", time.Duration(rusage.Stime.Nano()))
-	if runtime.GOOS == "darwin" {
-		// Darwin reports in bytes, Linux seems to report in KiB even
-		// though the manpage says otherwise.
-		rusage.Maxrss /= 1024
-	}
-	log.Println("MaxRSS:", rusage.Maxrss, "KiB")
-
-	fmt.Printf("%s,%d,%d,%d,%.02f,%.02f,%.02f,%d\n",
-		sv.Version,
-		ss.Alloc/1024,
-		ss.Sys/1024,
-		ss.Goroutines,
-		wallTime.Seconds(),
-		time.Duration(rusage.Utime.Nano()).Seconds(),
-		time.Duration(rusage.Stime.Nano()).Seconds(),
-		rusage.Maxrss)
-}

cmd/stcli/main.go

@@ -14,7 +14,6 @@ import (
 	"log"
 	"os"
 	"reflect"
-	"strings"
 
 	"github.com/AudriusButkevicius/recli"
 	"github.com/flynn-archive/go-shlex"
@@ -22,6 +21,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/urfave/cli"
@@ -85,7 +85,7 @@ func main() {
 		myID := protocol.NewDeviceID(cert.Certificate[0])
 
 		// Load the config
-		cfg, err := config.Load(locations.Get(locations.ConfigFile), myID)
+		cfg, err := config.Load(locations.Get(locations.ConfigFile), myID, events.NoopLogger)
 		if err != nil {
 			log.Fatalln(errors.Wrap(err, "loading config"))
 		}
@@ -127,7 +127,7 @@ func main() {
 	app.HelpName = app.Name
 	app.Author = "The Syncthing Authors"
 	app.Usage = "Syncthing command line interface"
-	app.Version = strings.Replace(build.LongVersion, "syncthing", app.Name, 1)
+	app.Version = build.Version
 	app.Flags = fakeFlags
 	app.Metadata = map[string]interface{}{
 		"client": client,

cmd/stcli/utils.go

@@ -12,7 +12,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"text/tabwriter"
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/urfave/cli"
@@ -45,12 +44,6 @@ func dumpOutput(url string) cli.ActionFunc {
 	}
 }
 
-func newTableWriter() *tabwriter.Writer {
-	writer := new(tabwriter.Writer)
-	writer.Init(os.Stdout, 0, 8, 0, '\t', 0)
-	return writer
-}
-
 func getConfig(c *APIClient) (config.Configuration, error) {
 	cfg := config.Configuration{}
 	response, err := c.Get("system/config")

cmd/stcrashreceiver/stcrashreceiver.go

@@ -13,6 +13,8 @@
 package main
 
 import (
+	"bytes"
+	"compress/gzip"
 	"flag"
 	"io"
 	"io/ioutil"
@@ -52,12 +54,12 @@ func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	// The final path component should be a SHA256 hash in hex, so 64 hex
 	// characters. We don't care about case on the request but use lower
 	// case internally.
-	base := strings.ToLower(path.Base(req.URL.Path))
-	if len(base) != 64 {
+	reportID := strings.ToLower(path.Base(req.URL.Path))
+	if len(reportID) != 64 {
 		http.Error(w, "Bad request", http.StatusBadRequest)
 		return
 	}
-	for _, c := range base {
+	for _, c := range reportID {
 		if c >= 'a' && c <= 'f' {
 			continue
 		}
@@ -68,40 +70,57 @@ func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
+	// The location of the report on disk, compressed
+	fullPath := filepath.Join(r.dir, r.dirFor(reportID), reportID) + ".gz"
+
 	switch req.Method {
+	case http.MethodGet:
+		r.serveGet(fullPath, w, req)
 	case http.MethodHead:
-		r.serveHead(base, w, req)
+		r.serveHead(fullPath, w, req)
 	case http.MethodPut:
-		r.servePut(base, w, req)
+		r.servePut(reportID, fullPath, w, req)
 	default:
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 	}
 }
 
+// serveGet responds to GET requests by serving the uncompressed report.
+func (r *crashReceiver) serveGet(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	fd, err := os.Open(fullPath)
+	if err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+		return
+	}
+
+	defer fd.Close()
+	gr, err := gzip.NewReader(fd)
+	if err != nil {
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+	_, _ = io.Copy(w, gr) // best effort
+}
+
 // serveHead responds to HEAD requests by checking if the named report
 // already exists in the system.
-func (r *crashReceiver) serveHead(base string, w http.ResponseWriter, _ *http.Request) {
-	path := filepath.Join(r.dirFor(base), base)
-	if _, err := os.Lstat(path); err != nil {
+func (r *crashReceiver) serveHead(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	if _, err := os.Lstat(fullPath); err != nil {
 		http.Error(w, "Not found", http.StatusNotFound)
 	}
-	// 200 OK
 }
 
 // servePut accepts and stores the given report.
-func (r *crashReceiver) servePut(base string, w http.ResponseWriter, req *http.Request) {
-	path := filepath.Join(r.dirFor(base), base)
-	fullPath := filepath.Join(r.dir, path)
-
+func (r *crashReceiver) servePut(reportID, fullPath string, w http.ResponseWriter, req *http.Request) {
 	// Ensure the destination directory exists
 	if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
-		log.Printf("Creating directory for report %s: %v", base, err)
+		log.Println("Creating directory:", err)
 		http.Error(w, "Internal server error", http.StatusInternalServerError)
 		return
 	}
 
 	// Read at most maxRequestSize of report data.
-	log.Println("Receiving report", base)
+	log.Println("Receiving report", reportID)
 	lr := io.LimitReader(req.Body, maxRequestSize)
 	bs, err := ioutil.ReadAll(lr)
 	if err != nil {
@@ -110,10 +129,16 @@ func (r *crashReceiver) servePut(base string, w http.ResponseWriter, req *http.R
 		return
 	}
 
-	// Create an output file
-	err = ioutil.WriteFile(fullPath, bs, 0644)
+	// Compress the report for storage
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	_, _ = gw.Write(bs) // can't fail
+	gw.Close()
+
+	// Create an output file with the compressed report
+	err = ioutil.WriteFile(fullPath, buf.Bytes(), 0644)
 	if err != nil {
-		log.Printf("Creating file for report %s: %v", base, err)
+		log.Println("Saving report:", err)
 		http.Error(w, "Internal server error", http.StatusInternalServerError)
 		return
 	}
@@ -122,7 +147,7 @@ func (r *crashReceiver) servePut(base string, w http.ResponseWriter, req *http.R
 	if r.dsn != "" {
 		go func() {
 			// There's no need for the client to have to wait for this part.
-			if err := sendReport(r.dsn, path, bs); err != nil {
+			if err := sendReport(r.dsn, reportID, bs); err != nil {
 				log.Println("Failed to send report:", err)
 			}
 		}()

cmd/stdiscosrv/apisrv.go

@@ -18,6 +18,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -279,6 +280,10 @@ func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addre
 		dbAddrs[i].Expires = expire
 	}
 
+	// The address slice must always be sorted for database merges to work
+	// properly.
+	sort.Sort(databaseAddressOrder(dbAddrs))
+
 	seen := now.UnixNano()
 	if s.repl != nil {
 		s.repl.send(key, dbAddrs, seen)
@@ -295,28 +300,66 @@ func certificateBytes(req *http.Request) []byte {
 		return req.TLS.PeerCertificates[0].Raw
 	}
 
+	var bs []byte
+
 	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
-		bs := []byte(hdr)
-		// The certificate is in PEM format but with spaces for newlines. We
-		// need to reinstate the newlines for the PEM decoder. But we need to
-		// leave the spaces in the BEGIN and END lines - the first and last
-		// space - alone.
-		firstSpace := bytes.Index(bs, []byte(" "))
-		lastSpace := bytes.LastIndex(bs, []byte(" "))
-		for i := firstSpace + 1; i < lastSpace; i++ {
-			if bs[i] == ' ' {
-				bs[i] = '\n'
+		if strings.Contains(hdr, "%") {
+			// Nginx using $ssl_client_escaped_cert
+			// The certificate is in PEM format with url encoding.
+			// We need to decode for the PEM decoder
+			hdr, err := url.QueryUnescape(hdr)
+			if err != nil {
+				// Decoding failed
+				return nil
+			}
+
+			bs = []byte(hdr)
+		} else {
+			// Nginx using $ssl_client_cert
+			// The certificate is in PEM format but with spaces for newlines. We
+			// need to reinstate the newlines for the PEM decoder. But we need to
+			// leave the spaces in the BEGIN and END lines - the first and last
+			// space - alone.
+			bs = []byte(hdr)
+			firstSpace := bytes.Index(bs, []byte(" "))
+			lastSpace := bytes.LastIndex(bs, []byte(" "))
+			for i := firstSpace + 1; i < lastSpace; i++ {
+				if bs[i] == ' ' {
+					bs[i] = '\n'
+				}
 			}
 		}
-		block, _ := pem.Decode(bs)
-		if block == nil {
+	} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
+		// Traefik 2 passtlsclientcert
+		// The certificate is in PEM format with url encoding but without newlines
+		// and start/end statements. We need to decode, reinstate the newlines every 64
+		// character and add statements for the PEM decoder
+		hdr, err := url.QueryUnescape(hdr)
+		if err != nil {
 			// Decoding failed
 			return nil
 		}
-		return block.Bytes
+
+		for i := 64; i < len(hdr); i += 65 {
+			hdr = hdr[:i] + "\n" + hdr[i:]
+		}
+
+		hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
+		hdr = hdr + "\n-----END CERTIFICATE-----\n"
+		bs = []byte(hdr)
 	}
 
-	return nil
+	if bs == nil {
+		return nil
+	}
+
+	block, _ := pem.Decode(bs)
+	if block == nil {
+		// Decoding failed
+		return nil
+	}
+
+	return block.Bytes
 }
 
 // fixupAddresses checks the list of addresses, removing invalid ones and

cmd/stdiscosrv/database.go

@@ -10,6 +10,7 @@
 package main
 
 import (
+	"log"
 	"sort"
 	"time"
 
@@ -263,12 +264,15 @@ func (s *levelDBStore) Stop() {
 // chosen for any duplicates.
 func merge(a, b DatabaseRecord) DatabaseRecord {
 	// Both lists must be sorted for this to work.
-	sort.Slice(a.Addresses, func(i, j int) bool {
-		return a.Addresses[i].Address < a.Addresses[j].Address
-	})
-	sort.Slice(b.Addresses, func(i, j int) bool {
-		return b.Addresses[i].Address < b.Addresses[j].Address
-	})
+	if !sort.IsSorted(databaseAddressOrder(a.Addresses)) {
+		log.Println("Warning: bug: addresses not correctly sorted in merge")
+		a.Addresses = sortedAddressCopy(a.Addresses)
+	}
+	if !sort.IsSorted(databaseAddressOrder(b.Addresses)) {
+		// no warning because this is the side we read from disk and it may
+		// legitimately predate correct sorting.
+		b.Addresses = sortedAddressCopy(b.Addresses)
+	}
 
 	res := DatabaseRecord{
 		Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
@@ -352,3 +356,24 @@ func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
 	}
 	return addrs
 }
+
+func sortedAddressCopy(addrs []DatabaseAddress) []DatabaseAddress {
+	sorted := make([]DatabaseAddress, len(addrs))
+	copy(sorted, addrs)
+	sort.Sort(databaseAddressOrder(sorted))
+	return sorted
+}
+
+type databaseAddressOrder []DatabaseAddress
+
+func (s databaseAddressOrder) Less(a, b int) bool {
+	return s[a].Address < s[b].Address
+}
+
+func (s databaseAddressOrder) Swap(a, b int) {
+	s[a], s[b] = s[b], s[a]
+}
+
+func (s databaseAddressOrder) Len() int {
+	return len(s)
+}

cmd/stdiscosrv/database.pb.go

@@ -3,12 +3,14 @@
 
 package main
 
-import proto "github.com/gogo/protobuf/proto"
-import fmt "fmt"
-import math "math"
-import _ "github.com/gogo/protobuf/gogoproto"
-
-import io "io"
+import (
+	fmt "fmt"
+	_ "github.com/gogo/protobuf/gogoproto"
+	proto "github.com/gogo/protobuf/proto"
+	io "io"
+	math "math"
+	math_bits "math/bits"
+)
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
@@ -19,7 +21,7 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
 
 type DatabaseRecord struct {
 	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
@@ -32,7 +34,7 @@ func (m *DatabaseRecord) Reset()         { *m = DatabaseRecord{} }
 func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
 func (*DatabaseRecord) ProtoMessage()    {}
 func (*DatabaseRecord) Descriptor() ([]byte, []int) {
-	return fileDescriptor_database_0f49e029703a04f5, []int{0}
+	return fileDescriptor_b90fe3356ea5df07, []int{0}
 }
 func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -42,15 +44,15 @@ func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, erro
 		return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalTo(b)
+		n, err := m.MarshalToSizedBuffer(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (dst *DatabaseRecord) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DatabaseRecord.Merge(dst, src)
+func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseRecord.Merge(m, src)
 }
 func (m *DatabaseRecord) XXX_Size() int {
 	return m.Size()
@@ -71,7 +73,7 @@ func (m *ReplicationRecord) Reset()         { *m = ReplicationRecord{} }
 func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
 func (*ReplicationRecord) ProtoMessage()    {}
 func (*ReplicationRecord) Descriptor() ([]byte, []int) {
-	return fileDescriptor_database_0f49e029703a04f5, []int{1}
+	return fileDescriptor_b90fe3356ea5df07, []int{1}
 }
 func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -81,15 +83,15 @@ func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, e
 		return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalTo(b)
+		n, err := m.MarshalToSizedBuffer(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (dst *ReplicationRecord) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ReplicationRecord.Merge(dst, src)
+func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ReplicationRecord.Merge(m, src)
 }
 func (m *ReplicationRecord) XXX_Size() int {
 	return m.Size()
@@ -109,7 +111,7 @@ func (m *DatabaseAddress) Reset()         { *m = DatabaseAddress{} }
 func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
 func (*DatabaseAddress) ProtoMessage()    {}
 func (*DatabaseAddress) Descriptor() ([]byte, []int) {
-	return fileDescriptor_database_0f49e029703a04f5, []int{2}
+	return fileDescriptor_b90fe3356ea5df07, []int{2}
 }
 func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -119,15 +121,15 @@ func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, err
 		return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalTo(b)
+		n, err := m.MarshalToSizedBuffer(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (dst *DatabaseAddress) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DatabaseAddress.Merge(dst, src)
+func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseAddress.Merge(m, src)
 }
 func (m *DatabaseAddress) XXX_Size() int {
 	return m.Size()
@@ -143,10 +145,34 @@ func init() {
 	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
 	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
 }
+
+func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
+
+var fileDescriptor_b90fe3356ea5df07 = []byte{
+	// 270 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
+	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
+	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
+	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
+	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
+	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
+	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
+	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
+	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
+	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
+	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
+	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
+	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
+	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
+	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
+	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
+	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
+}
+
 func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -154,44 +180,51 @@ func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Addresses) > 0 {
-		for _, msg := range m.Addresses {
-			dAtA[i] = 0xa
-			i++
-			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
-			n, err := msg.MarshalTo(dAtA[i:])
-			if err != nil {
-				return 0, err
-			}
-			i += n
-		}
-	}
-	if m.Misses != 0 {
-		dAtA[i] = 0x10
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+	if m.Missed != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+		i--
+		dAtA[i] = 0x20
 	}
 	if m.Seen != 0 {
-		dAtA[i] = 0x18
-		i++
 		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
 	}
-	if m.Missed != 0 {
-		dAtA[i] = 0x20
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+	if m.Misses != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+		i--
+		dAtA[i] = 0x10
 	}
-	return i, nil
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
 }
 
 func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -199,40 +232,48 @@ func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Key) > 0 {
-		dAtA[i] = 0xa
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
-		i += copy(dAtA[i:], m.Key)
+	if m.Seen != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
 	}
 	if len(m.Addresses) > 0 {
-		for _, msg := range m.Addresses {
-			dAtA[i] = 0x12
-			i++
-			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
-			n, err := msg.MarshalTo(dAtA[i:])
-			if err != nil {
-				return 0, err
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
 			}
-			i += n
+			i--
+			dAtA[i] = 0x12
 		}
 	}
-	if m.Seen != 0 {
-		dAtA[i] = 0x18
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+	if len(m.Key) > 0 {
+		i -= len(m.Key)
+		copy(dAtA[i:], m.Key)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		i--
+		dAtA[i] = 0xa
 	}
-	return i, nil
+	return len(dAtA) - i, nil
 }
 
 func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -240,32 +281,40 @@ func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Address) > 0 {
-		dAtA[i] = 0xa
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
-		i += copy(dAtA[i:], m.Address)
-	}
 	if m.Expires != 0 {
-		dAtA[i] = 0x10
-		i++
 		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
+		i--
+		dAtA[i] = 0x10
 	}
-	return i, nil
+	if len(m.Address) > 0 {
+		i -= len(m.Address)
+		copy(dAtA[i:], m.Address)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
 }
 
 func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
+	offset -= sovDatabase(v)
+	base := offset
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
 		v >>= 7
 		offset++
 	}
 	dAtA[offset] = uint8(v)
-	return offset + 1
+	return base
 }
 func (m *DatabaseRecord) Size() (n int) {
 	if m == nil {
@@ -330,14 +379,7 @@ func (m *DatabaseAddress) Size() (n int) {
 }
 
 func sovDatabase(x uint64) (n int) {
-	for {
-		n++
-		x >>= 7
-		if x == 0 {
-			break
-		}
-	}
-	return n
+	return (math_bits.Len64(x|1) + 6) / 7
 }
 func sozDatabase(x uint64) (n int) {
 	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
@@ -357,7 +399,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -385,7 +427,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= (int(b) & 0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -394,6 +436,9 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -416,7 +461,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Misses |= (int32(b) & 0x7F) << shift
+				m.Misses |= int32(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -435,7 +480,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= (int64(b) & 0x7F) << shift
+				m.Seen |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -454,7 +499,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Missed |= (int64(b) & 0x7F) << shift
+				m.Missed |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -468,6 +513,9 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -495,7 +543,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -523,7 +571,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -533,6 +581,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -552,7 +603,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= (int(b) & 0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -561,6 +612,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -583,7 +637,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= (int64(b) & 0x7F) << shift
+				m.Seen |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -597,6 +651,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -624,7 +681,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -652,7 +709,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -662,6 +719,9 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -681,7 +741,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Expires |= (int64(b) & 0x7F) << shift
+				m.Expires |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -695,6 +755,9 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -710,6 +773,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 func skipDatabase(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0
+	depth := 0
 	for iNdEx < l {
 		var wire uint64
 		for shift := uint(0); ; shift += 7 {
@@ -741,10 +805,8 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			return iNdEx, nil
 		case 1:
 			iNdEx += 8
-			return iNdEx, nil
 		case 2:
 			var length int
 			for shift := uint(0); ; shift += 7 {
@@ -761,76 +823,34 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			iNdEx += length
 			if length < 0 {
 				return 0, ErrInvalidLengthDatabase
 			}
-			return iNdEx, nil
+			iNdEx += length
 		case 3:
-			for {
-				var innerWire uint64
-				var start int = iNdEx
-				for shift := uint(0); ; shift += 7 {
-					if shift >= 64 {
-						return 0, ErrIntOverflowDatabase
-					}
-					if iNdEx >= l {
-						return 0, io.ErrUnexpectedEOF
-					}
-					b := dAtA[iNdEx]
-					iNdEx++
-					innerWire |= (uint64(b) & 0x7F) << shift
-					if b < 0x80 {
-						break
-					}
-				}
-				innerWireType := int(innerWire & 0x7)
-				if innerWireType == 4 {
-					break
-				}
-				next, err := skipDatabase(dAtA[start:])
-				if err != nil {
-					return 0, err
-				}
-				iNdEx = start + next
-			}
-			return iNdEx, nil
+			depth++
 		case 4:
-			return iNdEx, nil
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupDatabase
+			}
+			depth--
 		case 5:
 			iNdEx += 4
-			return iNdEx, nil
 		default:
 			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
 		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthDatabase
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
 	}
-	panic("unreachable")
+	return 0, io.ErrUnexpectedEOF
 }
 
 var (
-	ErrInvalidLengthDatabase = fmt.Errorf("proto: negative length found during unmarshaling")
-	ErrIntOverflowDatabase   = fmt.Errorf("proto: integer overflow")
+	ErrInvalidLengthDatabase        = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowDatabase          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupDatabase = fmt.Errorf("proto: unexpected end of group")
 )
-
-func init() { proto.RegisterFile("database.proto", fileDescriptor_database_0f49e029703a04f5) }
-
-var fileDescriptor_database_0f49e029703a04f5 = []byte{
-	// 270 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
-	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
-	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
-	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
-	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
-	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
-	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
-	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
-	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
-	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
-	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
-	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
-	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
-	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
-	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
-	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
-	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
-}

cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv

@@ -0,0 +1,4 @@
+[stdiscosrv]
+title=Syncthing discovery server
+description=Lets syncthing clients discover each other
+ports=8443/tcp

cmd/stdiscosrv/etc/linux-systemd/default

@@ -0,0 +1,3 @@
+# Default settings for syncthing-relaysrv (strelaysrv).
+## Add Options here:
+DISCOSRV_OPTS=

cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=Syncthing Discovery Server
+After=network.target
+Documentation=man:stdiscosrv(1)
+
+[Service]
+WorkingDirectory=/var/lib/syncthing-discosrv
+EnvironmentFile=/etc/default/syncthing-discosrv
+ExecStart=/usr/bin/stdiscosrv $DISCOSRV_OPTS
+
+# Hardening
+User=syncthing-discosrv
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-discosrv
+NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
+
+[Install]
+WantedBy=multi-user.target
+Alias=syncthing-discosrv.service

cmd/stdiscosrv/main.go

@@ -9,17 +9,15 @@ package main
 import (
 	"crypto/tls"
 	"flag"
-	"fmt"
 	"log"
 	"net"
 	"net/http"
 	"os"
-	"runtime"
-	"strconv"
 	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -65,24 +63,6 @@ var levelDBOptions = &opt.Options{
 	WriteBuffer: 32 << 20, // default 4<<20
 }
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`stdiscosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
-}
-
 var (
 	debug = false
 )
@@ -109,14 +89,18 @@ func main() {
 	flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
 	flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
 	flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
-	log.Println(LongVersion)
+	log.Println(build.LongVersion)
+	if *showVersion {
+		return
+	}
 
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}

cmd/stdiscosrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-discosrv --ingroup syncthing syncthing-discosrv

cmd/stfinddevice/main.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -82,7 +83,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 }
 
 func checkServer(deviceID protocol.DeviceID, server string) checkResult {
-	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger)
 	if err != nil {
 		return checkResult{error: err}
 	}

cmd/stindex/dump.go

@@ -13,11 +13,15 @@ import (
 	"time"
 
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func dump(ldb *db.Lowlevel) {
-	it := ldb.NewIterator(nil, nil)
+func dump(ldb backend.Backend) {
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {

cmd/stindex/dumpsize.go

@@ -10,8 +10,10 @@ import (
 	"container/heap"
 	"encoding/binary"
 	"fmt"
+	"log"
 
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
 )
 
 type SizedElement struct {
@@ -37,11 +39,14 @@ func (h *ElementHeap) Pop() interface{} {
 	return x
 }
 
-func dumpsize(ldb *db.Lowlevel) {
+func dumpsize(ldb backend.Backend) {
 	h := &ElementHeap{}
 	heap.Init(h)
 
-	it := ldb.NewIterator(nil, nil)
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
 	var ele SizedElement
 	for it.Next() {
 		key := it.Key()

cmd/stindex/idxck.go

@@ -10,8 +10,10 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
+	"log"
 
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -31,7 +33,7 @@ type sequenceKey struct {
 	sequence uint64
 }
 
-func idxck(ldb *db.Lowlevel) (success bool) {
+func idxck(ldb backend.Backend) (success bool) {
 	folders := make(map[uint32]string)
 	devices := make(map[uint32]string)
 	deviceToIDs := make(map[string]uint32)
@@ -42,7 +44,10 @@ func idxck(ldb *db.Lowlevel) (success bool) {
 	var localDeviceKey uint32
 	success = true
 
-	it := ldb.NewIterator(nil, nil)
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {

cmd/stindex/main.go

@@ -13,7 +13,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
 )
 
 func main() {
@@ -30,7 +30,7 @@ func main() {
 		path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
 	}
 
-	ldb, err := db.OpenRO(path)
+	ldb, err := backend.OpenLevelDBRO(path)
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/strelaypoolsrv/main.go

@@ -7,6 +7,7 @@ package main
 import (
 	"bytes"
 	"compress/gzip"
+	"context"
 	"crypto/tls"
 	"encoding/json"
 	"flag"
@@ -480,7 +481,7 @@ func handleRelayTest(request request) {
 	if debug {
 		log.Println("Request for", request.relay)
 	}
-	if !client.TestRelay(request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
+	if !client.TestRelay(context.TODO(), request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
 		if debug {
 			log.Println("Test for relay", request.relay, "failed")
 		}
@@ -633,7 +634,7 @@ func createTestCertificate() tls.Certificate {
 	}
 
 	certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
-	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv")
+	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 20*365)
 	if err != nil {
 		log.Fatalln("Failed to create test X509 key pair:", err)
 	}

cmd/strelaysrv/etc/firewall-ufw/strelaysrv

@@ -0,0 +1,9 @@
+[strelaysrv]
+title=Syncthing relay server
+description=Proxies traffic of syncthing client behind firewalls
+ports=22067/tcp
+
+[strelaysrv-metrics]
+title=Syncthing relay metrics
+description=Provides metrics about the syncthing relay server
+ports=22070/tcp

cmd/strelaysrv/etc/linux-systemd/default

@@ -0,0 +1,5 @@
+# Default settings for syncthing-relaysrv (strelaysrv).
+NAT=true
+
+## Add Options here:
+RELAYSRV_OPTS=

cmd/strelaysrv/etc/linux-systemd/strelaysrv.service

@@ -1,17 +1,25 @@
 [Unit]
-Description=Syncthing relay server
+Description=Syncthing Relay Server
 After=network.target
+Documentation=man:strelaysrv(1)
 
 [Service]
-User=strelaysrv
-Group=strelaysrv
-ExecStart=/usr/bin/strelaysrv
-WorkingDirectory=/var/lib/strelaysrv
+WorkingDirectory=/var/lib/syncthing-relaysrv
+EnvironmentFile=/etc/default/syncthing-relaysrv
+ExecStart=/usr/bin/strelaysrv -nat=${NAT} $RELAYSRV_OPTS
 
-PrivateTmp=true
-ProtectSystem=full
-ProtectHome=true
+# Hardening
+User=syncthing-relaysrv
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-relaysrv
 NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
 
 [Install]
 WantedBy=multi-user.target
+Alias=syncthing-relaysrv.service

cmd/strelaysrv/main.go

@@ -14,12 +14,13 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
 	"sync/atomic"
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/relay/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -33,24 +34,6 @@ import (
 	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
 )
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`strelaysrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
-}
-
 var (
 	listen string
 	debug  bool
@@ -116,8 +99,14 @@ func main() {
 	flag.IntVar(&natTimeout, "nat-timeout", 10, "NAT discovery timeout in seconds")
 	flag.BoolVar(&pprofEnabled, "pprof", false, "Enable the built in profiling on the status server")
 	flag.IntVar(&networkBufferSize, "network-buffer", 2048, "Network buffer size (two of these per proxied connection)")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
+	if *showVersion {
+		fmt.Println(build.LongVersion)
+		return
+	}
+
 	if extAddress == "" {
 		extAddress = listen
 	}
@@ -146,7 +135,7 @@ func main() {
 		}
 	}
 
-	log.Println(LongVersion)
+	log.Println(build.LongVersion)
 
 	maxDescriptors, err := osutil.MaximizeOpenFileLimit()
 	if maxDescriptors > 0 {
@@ -166,7 +155,7 @@ func main() {
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
@@ -194,7 +183,7 @@ func main() {
 		log.Println("ID:", id)
 	}
 
-	wrapper := config.Wrap("config", config.New(id))
+	wrapper := config.Wrap("config", config.New(id), events.NoopLogger)
 	wrapper.SetOptions(config.OptionsConfiguration{
 		NATLeaseM:   natLease,
 		NATRenewalM: natRenewal,

cmd/strelaysrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-relaysrv --ingroup syncthing syncthing-relaysrv

cmd/strelaysrv/status.go

@@ -10,6 +10,8 @@ import (
 	"runtime"
 	"sync/atomic"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/build"
 )
 
 var rc *rateCalculator
@@ -40,10 +42,10 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 
 	sessionMut.Lock()
 	// This can potentially be double the number of pending sessions, as each session has two keys, one for each side.
-	status["version"] = Version
-	status["buildHost"] = BuildHost
-	status["buildUser"] = BuildUser
-	status["buildDate"] = BuildDate
+	status["version"] = build.Version
+	status["buildHost"] = build.Host
+	status["buildUser"] = build.User
+	status["buildDate"] = build.Date
 	status["startTime"] = rc.startTime
 	status["uptimeSeconds"] = time.Since(rc.startTime) / time.Second
 	status["numPendingSessionKeys"] = len(pendingSessions)

cmd/strelaysrv/testutil/main.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"crypto/tls"
 	"flag"
 	"log"
@@ -19,6 +20,8 @@ import (
 )
 
 func main() {
+	ctx := context.Background()
+
 	log.SetOutput(os.Stdout)
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
 
@@ -76,7 +79,7 @@ func main() {
 		}()
 
 		for {
-			conn, err := client.JoinSession(<-recv)
+			conn, err := client.JoinSession(ctx, <-recv)
 			if err != nil {
 				log.Fatalln("Failed to join", err)
 			}
@@ -90,13 +93,13 @@ func main() {
 			log.Fatal(err)
 		}
 
-		invite, err := client.GetInvitationFromRelay(uri, id, []tls.Certificate{cert}, 10*time.Second)
+		invite, err := client.GetInvitationFromRelay(ctx, uri, id, []tls.Certificate{cert}, 10*time.Second)
 		if err != nil {
 			log.Fatal(err)
 		}
 
 		log.Println("Received invitation", invite)
-		conn, err := client.JoinSession(invite)
+		conn, err := client.JoinSession(ctx, invite)
 		if err != nil {
 			log.Fatalln("Failed to join", err)
 		}
@@ -104,7 +107,7 @@ func main() {
 		connectToStdio(stdin, conn)
 		log.Println("Finished", conn.RemoteAddr(), conn.LocalAddr())
 	} else if test {
-		if client.TestRelay(uri, []tls.Certificate{cert}, time.Second, 2*time.Second, 4) {
+		if client.TestRelay(ctx, uri, []tls.Certificate{cert}, time.Second, 2*time.Second, 4) {
 			log.Println("OK")
 		} else {
 			log.Println("FAIL")

cmd/stupgrades/main.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"os"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/upgrade"
+)
+
+const defaultURL = "https://api.github.com/repos/syncthing/syncthing/releases?per_page=25"
+
+func main() {
+	url := flag.String("u", defaultURL, "GitHub releases url")
+	flag.Parse()
+
+	rels := upgrade.FetchLatestReleases(*url, "")
+	if rels == nil {
+		// An error was already logged
+		os.Exit(1)
+	}
+
+	sort.Sort(upgrade.SortByRelease(rels))
+	rels = filterForLatest(rels)
+
+	if err := json.NewEncoder(os.Stdout).Encode(rels); err != nil {
+		os.Exit(1)
+	}
+}
+
+// filterForLatest returns the latest stable and prerelease only. If the
+// stable version is newer (comes first in the list) there is no need to go
+// looking for a prerelease at all.
+func filterForLatest(rels []upgrade.Release) []upgrade.Release {
+	var filtered []upgrade.Release
+	var havePre bool
+	for _, rel := range rels {
+		if !rel.Prerelease {
+			// We found a stable version, we're good now.
+			filtered = append(filtered, rel)
+			break
+		}
+		if rel.Prerelease && !havePre {
+			// We remember the first prerelease we find.
+			filtered = append(filtered, rel)
+			havePre = true
+		}
+	}
+	return filtered
+}

cmd/syncthing/blockprof.go

@@ -22,11 +22,15 @@ func init() {
 			panic("Couldn't find block profiler")
 		}
 		l.Debugln("Starting block profiling")
-		go saveBlockingProfiles(profiler)
+		go func() {
+			err := saveBlockingProfiles(profiler) // Only returns on error
+			l.Warnln("Block profiler failed:", err)
+			panic("Block profiler failed")
+		}()
 	}
 }
 
-func saveBlockingProfiles(profiler *pprof.Profile) {
+func saveBlockingProfiles(profiler *pprof.Profile) error {
 	runtime.SetBlockProfileRate(1)
 
 	t0 := time.Now()
@@ -35,16 +39,16 @@ func saveBlockingProfiles(profiler *pprof.Profile) {
 
 		fd, err := os.Create(fmt.Sprintf("block-%05d-%07d.pprof", syscall.Getpid(), startms))
 		if err != nil {
-			panic(err)
+			return err
 		}
 		err = profiler.WriteTo(fd, 0)
 		if err != nil {
-			panic(err)
+			return err
 		}
 		err = fd.Close()
 		if err != nil {
-			panic(err)
+			return err
 		}
-
 	}
+	return nil
 }

cmd/syncthing/debug.go

@@ -7,16 +7,9 @@
 package main
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
 	l = logger.DefaultLogger.NewFacility("main", "Main package")
 )
-
-func init() {
-	l.SetDebug("main", strings.Contains(os.Getenv("STTRACE"), "main") || os.Getenv("STTRACE") == "all")
-}

cmd/syncthing/heapprof.go

@@ -23,11 +23,15 @@ func init() {
 			rate = i
 		}
 		l.Debugln("Starting heap profiling")
-		go saveHeapProfiles(rate)
+		go func() {
+			err := saveHeapProfiles(rate) // Only returns on error
+			l.Warnln("Heap profiler failed:", err)
+			panic("Heap profiler failed")
+		}()
 	}
 }
 
-func saveHeapProfiles(rate int) {
+func saveHeapProfiles(rate int) error {
 	runtime.MemProfileRate = rate
 	var memstats, prevMemstats runtime.MemStats
 
@@ -38,21 +42,21 @@ func saveHeapProfiles(rate int) {
 		if memstats.HeapInuse > prevMemstats.HeapInuse {
 			fd, err := os.Create(name + ".tmp")
 			if err != nil {
-				panic(err)
+				return err
 			}
 			err = pprof.WriteHeapProfile(fd)
 			if err != nil {
-				panic(err)
+				return err
 			}
 			err = fd.Close()
 			if err != nil {
-				panic(err)
+				return err
 			}
 
 			os.Remove(name) // Error deliberately ignored
 			err = os.Rename(name+".tmp", name)
 			if err != nil {
-				panic(err)
+				return err
 			}
 
 			prevMemstats = memstats

cmd/syncthing/main.go

@@ -45,19 +45,8 @@ import (
 )
 
 const (
-	exitSuccess            = 0
-	exitError              = 1
-	exitNoUpgradeAvailable = 2
-	exitRestarting         = 3
-	exitUpgrading          = 4
-)
-
-const (
-	bepProtocolName      = "bep/1.0"
-	tlsDefaultCommonName = "syncthing"
-	maxSystemErrors      = 5
-	initialSystemLog     = 10
-	maxSystemLog         = 250
+	tlsDefaultCommonName   = "syncthing"
+	deviceCertLifetimeDays = 20 * 365
 )
 
 const (
@@ -165,6 +154,8 @@ type RuntimeOptions struct {
 	browserOnly      bool
 	hideConsole      bool
 	logFile          string
+	logMaxSize       int
+	logMaxFiles      int
 	auditEnabled     bool
 	auditFile        string
 	paused           bool
@@ -191,6 +182,8 @@ func defaultRuntimeOptions() RuntimeOptions {
 		cpuProfile:   os.Getenv("STCPUPROFILE") != "",
 		stRestarting: os.Getenv("STRESTART") != "",
 		logFlags:     log.Ltime,
+		logMaxSize:   10 << 20, // 10 MiB
+		logMaxFiles:  3,        // plus the current one
 	}
 
 	if os.Getenv("STTRACE") != "" {
@@ -233,6 +226,8 @@ func parseCommandLineOptions() RuntimeOptions {
 	flag.BoolVar(&options.paused, "paused", false, "Start with all devices and folders paused")
 	flag.BoolVar(&options.unpaused, "unpaused", false, "Start with all devices and folders unpaused")
 	flag.StringVar(&options.logFile, "logfile", options.logFile, "Log file name (still always logs to stdout). Cannot be used together with -no-restart/STNORESTART environment variable.")
+	flag.IntVar(&options.logMaxSize, "log-max-size", options.logMaxSize, "Maximum size of any file (zero to disable log rotation).")
+	flag.IntVar(&options.logMaxFiles, "log-max-old-files", options.logMaxFiles, "Number of old files to keep (zero to keep only current).")
 	flag.StringVar(&options.auditFile, "auditfile", options.auditFile, "Specify audit file (use \"-\" for stdout, \"--\" for stderr)")
 	flag.BoolVar(&options.allowNewerConfig, "allow-newer-config", false, "Allow loading newer than current config version")
 	if runtime.GOOS == "windows" {
@@ -271,7 +266,7 @@ func main() {
 	// default location
 	if options.noRestart && (options.logFile != "" && options.logFile != "-") {
 		l.Warnln("-logfile may not be used with -no-restart or STNORESTART")
-		os.Exit(exitError)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 
 	if options.hideConsole {
@@ -285,12 +280,12 @@ func main() {
 			options.confDir, err = filepath.Abs(options.confDir)
 			if err != nil {
 				l.Warnln("Failed to make options path absolute:", err)
-				os.Exit(exitError)
+				os.Exit(syncthing.ExitError.AsInt())
 			}
 		}
 		if err := locations.SetBaseDir(locations.ConfigBaseDir, options.confDir); err != nil {
 			l.Warnln(err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 	}
 
@@ -328,7 +323,7 @@ func main() {
 		)
 		if err != nil {
 			l.Warnln("Error reading device ID:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 
 		fmt.Println(protocol.NewDeviceID(cert.Certificate[0]))
@@ -338,7 +333,7 @@ func main() {
 	if options.browserOnly {
 		if err := openGUI(protocol.EmptyDeviceID); err != nil {
 			l.Warnln("Failed to open web UI:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		return
 	}
@@ -346,7 +341,7 @@ func main() {
 	if options.generateDir != "" {
 		if err := generate(options.generateDir); err != nil {
 			l.Warnln("Failed to generate config and keys:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		return
 	}
@@ -354,14 +349,14 @@ func main() {
 	// Ensure that our home directory exists.
 	if err := ensureDir(locations.GetBaseDir(locations.ConfigBaseDir), 0700); err != nil {
 		l.Warnln("Failure on home directory:", err)
-		os.Exit(exitError)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 
 	if options.upgradeTo != "" {
 		err := upgrade.ToURL(options.upgradeTo)
 		if err != nil {
 			l.Warnln("Error while Upgrading:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		l.Infoln("Upgraded from", options.upgradeTo)
 		return
@@ -381,7 +376,7 @@ func main() {
 	if options.resetDatabase {
 		if err := resetDB(); err != nil {
 			l.Warnln("Resetting database:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		return
 	}
@@ -394,7 +389,7 @@ func main() {
 }
 
 func openGUI(myID protocol.DeviceID) error {
-	cfg, err := loadOrDefaultConfig(myID)
+	cfg, err := loadOrDefaultConfig(myID, events.NoopLogger)
 	if err != nil {
 		return err
 	}
@@ -424,7 +419,7 @@ func generate(generateDir string) error {
 	if err == nil {
 		l.Warnln("Key exists; will not overwrite.")
 	} else {
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, tlsDefaultCommonName)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, tlsDefaultCommonName, deviceCertLifetimeDays)
 		if err != nil {
 			return errors.Wrap(err, "create certificate")
 		}
@@ -437,7 +432,7 @@ func generate(generateDir string) error {
 		l.Warnln("Config exists; will not overwrite.")
 		return nil
 	}
-	cfg, err := defaultConfig(cfgFile, myID)
+	cfg, err := syncthing.DefaultConfig(cfgFile, myID, events.NoopLogger, noDefaultFolder)
 	if err != nil {
 		return err
 	}
@@ -471,18 +466,18 @@ func debugFacilities() string {
 }
 
 func checkUpgrade() upgrade.Release {
-	cfg, _ := loadOrDefaultConfig(protocol.EmptyDeviceID)
+	cfg, _ := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger)
 	opts := cfg.Options()
 	release, err := upgrade.LatestRelease(opts.ReleasesURL, build.Version, opts.UpgradeToPreReleases)
 	if err != nil {
 		l.Warnln("Upgrade:", err)
-		os.Exit(exitError)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 
 	if upgrade.CompareVersions(release.Tag, build.Version) <= 0 {
 		noUpgradeMessage := "No upgrade available (current %q >= latest %q)."
 		l.Infof(noUpgradeMessage, build.Version, release.Tag)
-		os.Exit(exitNoUpgradeAvailable)
+		os.Exit(syncthing.ExitNoUpgradeAvailable.AsInt())
 	}
 
 	l.Infof("Upgrade available (current %q < latest %q)", build.Version, release.Tag)
@@ -491,12 +486,12 @@ func checkUpgrade() upgrade.Release {
 
 func performUpgrade(release upgrade.Release) {
 	// Use leveldb database locks to protect against concurrent upgrades
-	_, err := syncthing.OpenGoleveldb(locations.Get(locations.Database))
+	_, err := syncthing.OpenGoleveldb(locations.Get(locations.Database), config.TuningAuto)
 	if err == nil {
 		err = upgrade.To(release)
 		if err != nil {
 			l.Warnln("Upgrade:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		l.Infof("Upgraded to %q", release.Tag)
 	} else {
@@ -504,15 +499,15 @@ func performUpgrade(release upgrade.Release) {
 		err = upgradeViaRest()
 		if err != nil {
 			l.Warnln("Upgrade:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		l.Infoln("Syncthing upgrading")
-		os.Exit(exitUpgrading)
+		os.Exit(syncthing.ExitUpgrade.AsInt())
 	}
 }
 
 func upgradeViaRest() error {
-	cfg, _ := loadOrDefaultConfig(protocol.EmptyDeviceID)
+	cfg, _ := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger)
 	u, err := url.Parse(cfg.GUI().URL())
 	if err != nil {
 		return err
@@ -523,7 +518,7 @@ func upgradeViaRest() error {
 	r.Header.Set("X-API-Key", cfg.GUI().APIKey)
 
 	tr := &http.Transport{
-		Dial:            dialer.Dial,
+		DialContext:     dialer.DialContext,
 		Proxy:           http.ProxyFromEnvironment,
 		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 	}
@@ -557,28 +552,23 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	l.Infoln(build.LongVersion)
 
 	// Ensure that we have a certificate and key.
-	cert, err := tls.LoadX509KeyPair(
+	cert, err := syncthing.LoadOrGenerateCertificate(
 		locations.Get(locations.CertFile),
 		locations.Get(locations.KeyFile),
 	)
 	if err != nil {
-		l.Infof("Generating ECDSA key and certificate for %s...", tlsDefaultCommonName)
-		cert, err = tlsutil.NewCertificate(
-			locations.Get(locations.CertFile),
-			locations.Get(locations.KeyFile),
-			tlsDefaultCommonName,
-		)
-		if err != nil {
-			l.Warnln("Failed to generate certificate:", err)
-			os.Exit(1)
-		}
+		l.Warnln("Failed to load/generate certificate:", err)
+		os.Exit(1)
 	}
-	myID := protocol.NewDeviceID(cert.Certificate[0])
 
-	cfg, err := loadConfigAtStartup(runtimeOptions.allowNewerConfig, myID)
+	evLogger := events.NewLogger()
+	go evLogger.Serve()
+	defer evLogger.Stop()
+
+	cfg, err := syncthing.LoadConfigAtStartup(locations.Get(locations.ConfigFile), cert, evLogger, runtimeOptions.allowNewerConfig, noDefaultFolder)
 	if err != nil {
 		l.Warnln("Failed to initialize config:", err)
-		os.Exit(exitError)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 
 	if runtimeOptions.unpaused {
@@ -588,7 +578,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	}
 
 	dbFile := locations.Get(locations.Database)
-	ldb, err := syncthing.OpenGoleveldb(dbFile)
+	ldb, err := syncthing.OpenGoleveldb(dbFile, cfg.Options().DatabaseTuning)
 	if err != nil {
 		l.Warnln("Error opening database:", err)
 		os.Exit(1)
@@ -603,7 +593,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		appOpts.DeadlockTimeoutS = secs
 	}
 
-	app := syncthing.New(cfg, ldb, cert, appOpts)
+	app := syncthing.New(cfg, ldb, evLogger, cert, appOpts)
 
 	setupSignalHandling(app)
 
@@ -615,11 +605,11 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		f, err := os.Create(fmt.Sprintf("cpu-%d.pprof", os.Getpid()))
 		if err != nil {
 			l.Warnln("Creating profile:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		if err := pprof.StartCPUProfile(f); err != nil {
 			l.Warnln("Starting profile:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 	}
 
@@ -648,11 +638,13 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		if runtimeOptions.NoUpgrade {
 			l.Infof("No automatic upgrades; STNOUPGRADE environment variable defined.")
 		} else {
-			go autoUpgrade(cfg, app)
+			go autoUpgrade(cfg, app, evLogger)
 		}
 	}
 
-	app.Start()
+	if err := app.Start(); err != nil {
+		os.Exit(syncthing.ExitError.AsInt())
+	}
 
 	cleanConfigDirectory()
 
@@ -693,79 +685,17 @@ func setupSignalHandling(app *syncthing.App) {
 	}()
 }
 
-func loadOrDefaultConfig(myID protocol.DeviceID) (config.Wrapper, error) {
+func loadOrDefaultConfig(myID protocol.DeviceID, evLogger events.Logger) (config.Wrapper, error) {
 	cfgFile := locations.Get(locations.ConfigFile)
-	cfg, err := config.Load(cfgFile, myID)
+	cfg, err := config.Load(cfgFile, myID, evLogger)
 
 	if err != nil {
-		cfg, err = defaultConfig(cfgFile, myID)
+		cfg, err = syncthing.DefaultConfig(cfgFile, myID, evLogger, noDefaultFolder)
 	}
 
 	return cfg, err
 }
 
-func loadConfigAtStartup(allowNewerConfig bool, myID protocol.DeviceID) (config.Wrapper, error) {
-	cfgFile := locations.Get(locations.ConfigFile)
-	cfg, err := config.Load(cfgFile, myID)
-	if os.IsNotExist(err) {
-		cfg, err = defaultConfig(cfgFile, myID)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to generate default config")
-		}
-		err = cfg.Save()
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to save default config")
-		}
-		l.Infof("Default config saved. Edit %s to taste (with Syncthing stopped) or use the GUI", cfg.ConfigPath())
-	} else if err == io.EOF {
-		return nil, errors.New("Failed to load config: unexpected end of file. Truncated or empty configuration?")
-	} else if err != nil {
-		return nil, errors.Wrap(err, "failed to load config")
-	}
-
-	if cfg.RawCopy().OriginalVersion != config.CurrentVersion {
-		if cfg.RawCopy().OriginalVersion == config.CurrentVersion+1101 {
-			l.Infof("Now, THAT's what we call a config from the future! Don't worry. As long as you hit that wire with the connecting hook at precisely eighty-eight miles per hour the instant the lightning strikes the tower... everything will be fine.")
-		}
-		if cfg.RawCopy().OriginalVersion > config.CurrentVersion && !allowNewerConfig {
-			return nil, fmt.Errorf("Config file version (%d) is newer than supported version (%d). If this is expected, use -allow-newer-config to override.", cfg.RawCopy().OriginalVersion, config.CurrentVersion)
-		}
-		err = archiveAndSaveConfig(cfg)
-		if err != nil {
-			return nil, errors.Wrap(err, "config archive")
-		}
-	}
-
-	return cfg, nil
-}
-
-func archiveAndSaveConfig(cfg config.Wrapper) error {
-	// Copy the existing config to an archive copy
-	archivePath := cfg.ConfigPath() + fmt.Sprintf(".v%d", cfg.RawCopy().OriginalVersion)
-	l.Infoln("Archiving a copy of old config file format at:", archivePath)
-	if err := copyFile(cfg.ConfigPath(), archivePath); err != nil {
-		return err
-	}
-
-	// Do a regular atomic config sve
-	return cfg.Save()
-}
-
-func copyFile(src, dst string) error {
-	bs, err := ioutil.ReadFile(src)
-	if err != nil {
-		return err
-	}
-
-	if err := ioutil.WriteFile(dst, bs, 0600); err != nil {
-		// Attempt to clean up
-		os.Remove(dst)
-		return err
-	}
-
-	return nil
-}
-
 func auditWriter(auditFile string) io.Writer {
 	var fd io.Writer
 	var err error
@@ -788,7 +718,7 @@ func auditWriter(auditFile string) io.Writer {
 		fd, err = os.OpenFile(auditFile, auditFlags, 0600)
 		if err != nil {
 			l.Warnln("Audit:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		auditDest = auditFile
 	}
@@ -798,22 +728,6 @@ func auditWriter(auditFile string) io.Writer {
 	return fd
 }
 
-func defaultConfig(cfgFile string, myID protocol.DeviceID) (config.Wrapper, error) {
-	newCfg, err := config.NewWithFreePorts(myID)
-	if err != nil {
-		return nil, err
-	}
-
-	if noDefaultFolder {
-		l.Infoln("We will skip creation of a default folder on first start since the proper envvar is set")
-		return config.Wrap(cfgFile, newCfg), nil
-	}
-
-	newCfg.Folders = append(newCfg.Folders, config.NewFolderConfiguration(myID, "default", "Default Folder", fs.FilesystemTypeBasic, locations.Get(locations.DefFolder)))
-	l.Infoln("Default folder created and/or linked to new config")
-	return config.Wrap(cfgFile, newCfg), nil
-}
-
 func resetDB() error {
 	return os.RemoveAll(locations.Get(locations.Database))
 }
@@ -861,9 +775,9 @@ func standbyMonitor(app *syncthing.App) {
 	}
 }
 
-func autoUpgrade(cfg config.Wrapper, app *syncthing.App) {
+func autoUpgrade(cfg config.Wrapper, app *syncthing.App, evLogger events.Logger) {
 	timer := time.NewTimer(0)
-	sub := events.Default.Subscribe(events.DeviceConnected)
+	sub := evLogger.Subscribe(events.DeviceConnected)
 	for {
 		select {
 		case event := <-sub.C():
@@ -885,7 +799,7 @@ func autoUpgrade(cfg config.Wrapper, app *syncthing.App) {
 
 		rel, err := upgrade.LatestRelease(opts.ReleasesURL, build.Version, opts.UpgradeToPreReleases)
 		if err == upgrade.ErrUpgradeUnsupported {
-			events.Default.Unsubscribe(sub)
+			sub.Unsubscribe()
 			return
 		}
 		if err != nil {
@@ -909,7 +823,7 @@ func autoUpgrade(cfg config.Wrapper, app *syncthing.App) {
 			timer.Reset(checkInterval)
 			continue
 		}
-		events.Default.Unsubscribe(sub)
+		sub.Unsubscribe()
 		l.Warnf("Automatically upgraded to version %q. Restarting in 1 minute.", rel.Tag)
 		time.Sleep(time.Minute)
 		app.Stop(syncthing.ExitUpgrade)
@@ -980,6 +894,6 @@ func setPauseState(cfg config.Wrapper, paused bool) {
 	}
 	if _, err := cfg.Replace(raw); err != nil {
 		l.Warnln("Cannot adjust paused state:", err)
-		os.Exit(exitError)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 }

cmd/syncthing/monitor.go

@@ -9,19 +9,23 @@ package main
 import (
 	"bufio"
 	"context"
+	"fmt"
 	"io"
 	"os"
 	"os/exec"
 	"os/signal"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/syncthing"
 )
 
 var (
@@ -46,7 +50,15 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 
 	logFile := runtimeOptions.logFile
 	if logFile != "-" {
-		var fileDst io.Writer = newAutoclosedFile(logFile, logFileAutoCloseDelay, logFileMaxOpenTime)
+		var fileDst io.Writer
+		if runtimeOptions.logMaxSize > 0 {
+			open := func(name string) (io.WriteCloser, error) {
+				return newAutoclosedFile(name, logFileAutoCloseDelay, logFileMaxOpenTime), nil
+			}
+			fileDst = newRotatedFile(logFile, open, int64(runtimeOptions.logMaxSize), runtimeOptions.logMaxFiles)
+		} else {
+			fileDst = newAutoclosedFile(logFile, logFileAutoCloseDelay, logFileMaxOpenTime)
+		}
 
 		if runtime.GOOS == "windows" {
 			// Translate line breaks to Windows standard
@@ -80,7 +92,7 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 
 		if t := time.Since(restarts[0]); t < loopThreshold {
 			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 
 		copy(restarts[0:], restarts[1:])
@@ -102,7 +114,8 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 		l.Infoln("Starting syncthing")
 		err = cmd.Start()
 		if err != nil {
-			panic(err)
+			l.Warnln("Error starting the main Syncthing process:", err)
+			panic("Error starting the main Syncthing process")
 		}
 
 		stdoutMut.Lock()
@@ -148,17 +161,14 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 				// Successful exit indicates an intentional shutdown
 				return
 			} else if exiterr, ok := err.(*exec.ExitError); ok {
-				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
-					switch status.ExitStatus() {
-					case exitUpgrading:
-						// Restart the monitor process to release the .old
-						// binary as part of the upgrade process.
-						l.Infoln("Restarting monitor...")
-						if err = restartMonitor(args); err != nil {
-							l.Warnln("Restart:", err)
-						}
-						return
+				if exiterr.ExitCode() == syncthing.ExitUpgrade.AsInt() {
+					// Restart the monitor process to release the .old
+					// binary as part of the upgrade process.
+					l.Infoln("Restarting monitor...")
+					if err = restartMonitor(args); err != nil {
+						l.Warnln("Restart:", err)
 					}
+					return
 				}
 			}
 		}
@@ -317,6 +327,81 @@ func restartMonitorWindows(args []string) error {
 	return cmd.Start()
 }
 
+// rotatedFile keeps a set of rotating logs. There will be the base file plus up
+// to maxFiles rotated ones, each ~ maxSize bytes large.
+type rotatedFile struct {
+	name        string
+	create      createFn
+	maxSize     int64 // bytes
+	maxFiles    int
+	currentFile io.WriteCloser
+	currentSize int64
+}
+
+// the createFn should act equivalently to os.Create
+type createFn func(name string) (io.WriteCloser, error)
+
+func newRotatedFile(name string, create createFn, maxSize int64, maxFiles int) *rotatedFile {
+	return &rotatedFile{
+		name:     name,
+		create:   create,
+		maxSize:  maxSize,
+		maxFiles: maxFiles,
+	}
+}
+
+func (r *rotatedFile) Write(bs []byte) (int, error) {
+	// Check if we're about to exceed the max size, and if so close this
+	// file so we'll start on a new one.
+	if r.currentSize+int64(len(bs)) > r.maxSize {
+		r.currentFile.Close()
+		r.currentFile = nil
+		r.currentSize = 0
+	}
+
+	// If we have no current log, rotate old files out of the way and create
+	// a new one.
+	if r.currentFile == nil {
+		r.rotate()
+		fd, err := r.create(r.name)
+		if err != nil {
+			return 0, err
+		}
+		r.currentFile = fd
+	}
+
+	n, err := r.currentFile.Write(bs)
+	r.currentSize += int64(n)
+	return n, err
+}
+
+func (r *rotatedFile) rotate() {
+	// The files are named "name", "name.0", "name.1", ...
+	// "name.(r.maxFiles-1)". Increase the numbers on the
+	// suffixed ones.
+	for i := r.maxFiles - 1; i > 0; i-- {
+		from := numberedFile(r.name, i-1)
+		to := numberedFile(r.name, i)
+		err := os.Rename(from, to)
+		if err != nil && !os.IsNotExist(err) {
+			fmt.Println("LOG: Rotating logs:", err)
+		}
+	}
+
+	// Rename the base to base.0
+	err := os.Rename(r.name, numberedFile(r.name, 0))
+	if err != nil && !os.IsNotExist(err) {
+		fmt.Println("LOG: Rotating logs:", err)
+	}
+}
+
+// numberedFile adds the number between the file name and the extension.
+func numberedFile(name string, num int) string {
+	ext := filepath.Ext(name) // contains the dot
+	withoutExt := name[:len(name)-len(ext)]
+	return fmt.Sprintf("%s.%d%s", withoutExt, num, ext)
+}
+
 // An autoclosedFile is an io.WriteCloser that opens itself for appending on
 // Write() and closes itself after an interval of no writes (closeDelay) or
 // when the file has been open for too long (maxOpenTime). A call to Write()
@@ -449,7 +534,7 @@ func childEnv() []string {
 // panicUploadMaxWait uploading panics...
 func maybeReportPanics() {
 	// Try to get a config to see if/where panics should be reported.
-	cfg, err := loadOrDefaultConfig(protocol.EmptyDeviceID)
+	cfg, err := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger)
 	if err != nil {
 		l.Warnln("Couldn't load config; not reporting crash")
 		return

cmd/syncthing/monitor_test.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -14,6 +15,123 @@ import (
 	"time"
 )
 
+func TestRotatedFile(t *testing.T) {
+	// Verify that log rotation happens.
+
+	dir, err := ioutil.TempDir("", "syncthing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	open := func(name string) (io.WriteCloser, error) {
+		return os.Create(name)
+	}
+
+	logName := filepath.Join(dir, "log.txt")
+	testData := []byte("12345678\n")
+	maxSize := int64(len(testData) + len(testData)/2)
+
+	// We allow the log file plus two rotated copies.
+	rf := newRotatedFile(logName, open, maxSize, 2)
+
+	// Write some bytes.
+	if _, err := rf.Write(testData); err != nil {
+		t.Fatal(err)
+	}
+	// They should be in the log.
+	checkSize(t, logName, len(testData))
+	checkNotExist(t, logName+".0")
+
+	// Write some more bytes. We should rotate and write into a new file as the
+	// new bytes don't fit.
+	if _, err := rf.Write(testData); err != nil {
+		t.Fatal(err)
+	}
+	checkSize(t, logName, len(testData))
+	checkSize(t, numberedFile(logName, 0), len(testData))
+	checkNotExist(t, logName+".1")
+
+	// Write another byte. That should fit without causing an extra rotate.
+	_, _ = rf.Write([]byte{42})
+	checkSize(t, logName, len(testData)+1)
+	checkSize(t, numberedFile(logName, 0), len(testData))
+	checkNotExist(t, numberedFile(logName, 1))
+
+	// Write some more bytes. We should rotate and write into a new file as the
+	// new bytes don't fit.
+	if _, err := rf.Write(testData); err != nil {
+		t.Fatal(err)
+	}
+	checkSize(t, logName, len(testData))
+	checkSize(t, numberedFile(logName, 0), len(testData)+1) // the one we wrote extra to, now rotated
+	checkSize(t, numberedFile(logName, 1), len(testData))
+	checkNotExist(t, numberedFile(logName, 2))
+
+	// Write some more bytes. We should rotate and write into a new file as the
+	// new bytes don't fit.
+	if _, err := rf.Write(testData); err != nil {
+		t.Fatal(err)
+	}
+	checkSize(t, logName, len(testData))
+	checkSize(t, numberedFile(logName, 0), len(testData))
+	checkSize(t, numberedFile(logName, 1), len(testData)+1)
+	checkNotExist(t, numberedFile(logName, 2)) // exceeds maxFiles so deleted
+}
+
+func TestNumberedFile(t *testing.T) {
+	// Mostly just illustrates where the number ends up and makes sure it
+	// doesn't crash without an extension.
+
+	cases := []struct {
+		in  string
+		num int
+		out string
+	}{
+		{
+			in:  "syncthing.log",
+			num: 42,
+			out: "syncthing.42.log",
+		},
+		{
+			in:  filepath.Join("asdfasdf", "syncthing.log.txt"),
+			num: 42,
+			out: filepath.Join("asdfasdf", "syncthing.log.42.txt"),
+		},
+		{
+			in:  "syncthing-log",
+			num: 42,
+			out: "syncthing-log.42",
+		},
+	}
+
+	for _, tc := range cases {
+		res := numberedFile(tc.in, tc.num)
+		if res != tc.out {
+			t.Errorf("numberedFile(%q, %d) => %q, expected %q", tc.in, tc.num, res, tc.out)
+		}
+	}
+}
+
+func checkSize(t *testing.T, name string, size int) {
+	t.Helper()
+	info, err := os.Lstat(name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.Size() != int64(size) {
+		t.Errorf("%s wrong size: %d != expected %d", name, info.Size(), size)
+	}
+}
+
+func checkNotExist(t *testing.T, name string) {
+	t.Helper()
+	_, err := os.Lstat(name)
+	if !os.IsNotExist(err) {
+		t.Errorf("%s should not exist", name)
+	}
+}
+
 func TestAutoClosedFile(t *testing.T) {
 	os.RemoveAll("_autoclose")
 	defer os.RemoveAll("_autoclose")

cmd/todos/main.go

@@ -1,39 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build ignore
-
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"os"
-)
-
-func main() {
-	buf := make([]byte, 4096)
-	var err error
-	for err == nil {
-		n, err := io.ReadFull(os.Stdin, buf)
-		if n > 0 {
-			buf = buf[:n]
-			repl := bytes.Replace(buf, []byte("\n"), []byte("\r\n"), -1)
-			_, err = os.Stdout.Write(repl)
-			if err != nil {
-				fmt.Println(err)
-				os.Exit(1)
-			}
-		}
-		if err == io.EOF {
-			return
-		}
-		buf = buf[:cap(buf)]
-	}
-	fmt.Println(err)
-	os.Exit(1)
-}

cmd/ursrv/main.go

@@ -755,6 +755,8 @@ func main() {
 	http.HandleFunc("/blockstats.json", withDB(db, blockStatsHandler))
 	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
 
+	go cacheRefresher(db)
+
 	err = srv.Serve(listener)
 	if err != nil {
 		log.Fatalln("https:", err)
@@ -767,7 +769,31 @@ var (
 	cacheMut  sync.Mutex
 )
 
-const maxCacheTime = 5 * 60 * time.Second
+const maxCacheTime = 15 * time.Minute
+
+func cacheRefresher(db *sql.DB) {
+	ticker := time.NewTicker(maxCacheTime - time.Minute)
+	defer ticker.Stop()
+	for range ticker.C {
+		cacheMut.Lock()
+		if err := refreshCacheLocked(db); err != nil {
+			log.Println(err)
+		}
+		cacheMut.Unlock()
+	}
+}
+
+func refreshCacheLocked(db *sql.DB) error {
+	rep := getReport(db)
+	buf := new(bytes.Buffer)
+	err := tpl.Execute(buf, rep)
+	if err != nil {
+		return err
+	}
+	cacheData = buf.Bytes()
+	cacheTime = time.Now()
+	return nil
+}
 
 func rootHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
 	if r.URL.Path == "/" || r.URL.Path == "/index.html" {
@@ -775,16 +801,11 @@ func rootHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
 		defer cacheMut.Unlock()
 
 		if time.Since(cacheTime) > maxCacheTime {
-			rep := getReport(db)
-			buf := new(bytes.Buffer)
-			err := tpl.Execute(buf, rep)
-			if err != nil {
+			if err := refreshCacheLocked(db); err != nil {
 				log.Println(err)
 				http.Error(w, "Template Error", http.StatusInternalServerError)
 				return
 			}
-			cacheData = buf.Bytes()
-			cacheTime = time.Now()
 		}
 
 		w.Header().Set("Content-Type", "text/html; charset=utf-8")
@@ -1395,7 +1416,7 @@ func getReport(db *sql.DB) map[string]interface{} {
 	r["categories"] = categories
 	r["versions"] = group(byVersion, analyticsFor(versions, 2000), 10)
 	r["versionPenetrations"] = penetrationLevels(analyticsFor(versions, 2000), []float64{50, 75, 90, 95})
-	r["platforms"] = group(byPlatform, analyticsFor(platforms, 2000), 5)
+	r["platforms"] = group(byPlatform, analyticsFor(platforms, 2000), 10)
 	r["compilers"] = group(byCompiler, analyticsFor(compilers, 2000), 5)
 	r["builders"] = analyticsFor(builders, 12)
 	r["distributions"] = analyticsFor(distributions, 10)

go.mod

@@ -4,49 +4,50 @@ require (
 	github.com/AudriusButkevicius/go-nat-pmp v0.0.0-20160522074932-452c97607362
 	github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6
 	github.com/AudriusButkevicius/recli v0.0.5
+	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e
-	github.com/calmh/du v1.0.1
 	github.com/calmh/xdr v1.1.0
 	github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d
-	github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 // indirect
+	github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67 // indirect
 	github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5
 	github.com/d4l3k/messagediff v1.2.1
 	github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
 	github.com/getsentry/raven-go v0.2.0
-	github.com/gobwas/glob v0.0.0-20170212200151-51eb1ee00b6d
-	github.com/gogo/protobuf v1.2.1
-	github.com/golang/groupcache v0.0.0-20171101203131-84a468cf14b4
-	github.com/jackpal/gateway v0.0.0-20161225004348-5795ac81146e
-	github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657
+	github.com/go-ole/go-ole v1.2.4 // indirect
+	github.com/gobwas/glob v0.2.3
+	github.com/gogo/protobuf v1.3.1
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6
+	github.com/golang/mock v1.3.1 // indirect
+	github.com/jackpal/gateway v1.0.5
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/lib/pq v1.2.0
-	github.com/lucas-clemente/quic-go v0.11.2
+	github.com/lucas-clemente/quic-go v0.12.1
 	github.com/maruel/panicparse v1.3.0
-	github.com/mattn/go-isatty v0.0.7
-	github.com/minio/sha256-simd v0.0.0-20190117184323-cc1980cb0338
-	github.com/onsi/ginkgo v1.8.0 // indirect
-	github.com/onsi/gomega v1.5.0 // indirect
+	github.com/mattn/go-isatty v0.0.10
+	github.com/minio/sha256-simd v0.1.1
+	github.com/onsi/ginkgo v1.9.0 // indirect
+	github.com/onsi/gomega v1.6.0 // indirect
 	github.com/oschwald/geoip2-golang v1.3.0
-	github.com/oschwald/maxminddb-golang v0.0.0-20170901134056-26fe5ace1c70 // indirect
-	github.com/petermattis/goid v0.0.0-20170816195418-3db12ebb2a59 // indirect
+	github.com/oschwald/maxminddb-golang v1.4.0 // indirect
+	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
 	github.com/pkg/errors v0.8.1
-	github.com/prometheus/client_golang v0.9.4
-	github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9
+	github.com/prometheus/client_golang v1.2.1
+	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563
 	github.com/sasha-s/go-deadlock v0.2.0
+	github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf
 	github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2
-	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
+	github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d
 	github.com/thejerf/suture v3.0.2+incompatible
-	github.com/urfave/cli v1.20.0
+	github.com/urfave/cli v1.22.2
 	github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0
-	golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8
-	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
-	golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect
+	golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472
+	golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297
 	golang.org/x/text v0.3.2
-	golang.org/x/time v0.0.0-20170927054726-6dc17368e09b
-	gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225 // indirect
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
+	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/ldap.v2 v2.5.1
-	gopkg.in/yaml.v2 v2.2.2 // indirect
 )
 
 go 1.12

go.sum

@@ -4,26 +4,37 @@ github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6 h1:Apvc
 github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6/go.mod h1:1N0EEx/irz4B1qV17wW82TFbjQrE7oX316Cki6eDY0Q=
 github.com/AudriusButkevicius/recli v0.0.5 h1:xUa55PvWTHBm17T6RvjElRO3y5tALpdceH86vhzQ5wg=
 github.com/AudriusButkevicius/recli v0.0.5/go.mod h1:Q2E26yc6RvWWEz/TJ/goUp6yXvipYdJI096hpoaqsNs=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e h1:2augTYh6E+XoNrrivZJBadpThP/dsvYKj0nzqfQ8tM4=
 github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
-github.com/calmh/du v1.0.1 h1:uDCrDbXVVPrzxSNRkpj6nqSfwrl5uRWH3zvrJgl7RRo=
-github.com/calmh/du v1.0.1/go.mod h1:pHNccp4cXQeyDaiV3S7t5GN+eGOgynF0VSLxJjk9tLU=
 github.com/calmh/xdr v1.1.0 h1:U/Dd4CXNLoo8EiQ4ulJUXkgO1/EyQLgDKLgpY1SOoJE=
 github.com/calmh/xdr v1.1.0/go.mod h1:E8sz2ByAdXC8MbANf1LCRYzedSnnc+/sXXJs/PVqoeg=
 github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d h1:As4937T5NVbJ/DmZT9z33pyLEprMd6CUSfhbmMY57Io=
 github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d/go.mod h1:3FK1bMar37f7jqVY7q/63k3OMX1c47pGCufzt3X0sYE=
-github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 h1:UNOqI3EKhvbqV8f1Vm3NIwkrhq388sGCeAH2Op7w0rc=
-github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
+github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67 h1:8k9FLYBLKT+9v2HQJ/a95ZemmTx+/ltJcAiRhVushG8=
+github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
+github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
+github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5 h1:Wg96Dh0MLTanEaPO0OkGtUIaa2jOnShAIOVUIzRHUxo=
 github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U=
 github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -36,32 +47,49 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
 github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
+github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gobwas/glob v0.0.0-20170212200151-51eb1ee00b6d h1:IngNQgbqr5ZOU0exk395Szrvkzes9Ilk1fmJfkw7d+M=
-github.com/gobwas/glob v0.0.0-20170212200151-51eb1ee00b6d/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/golang/groupcache v0.0.0-20171101203131-84a468cf14b4 h1:6o8aP0LGMKzo3NzwhhX6EJsiJ3ejmj+9yA/3p8Fjjlw=
-github.com/golang/groupcache v0.0.0-20171101203131-84a468cf14b4/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
+github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/jackpal/gateway v0.0.0-20161225004348-5795ac81146e h1:lS8IitpqG4RkZbEDlZg5Z7FvBdWLVjSVfsPGOKafEkI=
-github.com/jackpal/gateway v0.0.0-20161225004348-5795ac81146e/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
+github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc=
+github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657 h1:vE7J1m7cCpiRVEIr1B5ccDxRpbPsWT5JU3if2Di5nE4=
-github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -70,16 +98,15 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
-github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lucas-clemente/quic-go v0.11.2 h1:Mop0ac3zALaBR3wGs6j8OYe/tcFvFsxTUFMkE/7yUOI=
-github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
-github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA=
-github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
-github.com/maruel/panicparse v1.2.1 h1:mNlHGiakrixj+AwF/qRpTwnj+zsWYPRLQ7wRqnJsfO0=
-github.com/maruel/panicparse v1.2.1/go.mod h1:vszMjr5QQ4F5FSRfraldcIA/BCw5xrdLL+zEcU2nRBs=
+github.com/lucas-clemente/quic-go v0.12.0 h1:TRbvZ6F++sofeGbh+Z2IIyIOhl8KyGnYuA06g2yrHdI=
+github.com/lucas-clemente/quic-go v0.12.0/go.mod h1:UXJJPE4RfFef/xPO5wQm0tITK8gNfqwTxjbE7s3Vb8s=
+github.com/lucas-clemente/quic-go v0.12.1 h1:BPITli+6KnKogtTxBk2aS4okr5dUHz2LtIDAP1b8UL4=
+github.com/lucas-clemente/quic-go v0.12.1/go.mod h1:UXJJPE4RfFef/xPO5wQm0tITK8gNfqwTxjbE7s3Vb8s=
+github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
+github.com/marten-seemann/qtls v0.3.2 h1:O7awy4bHEzSX/K3h+fZig3/Vo03s/RxlxgsAk9sYamI=
+github.com/marten-seemann/qtls v0.3.2/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
 github.com/maruel/panicparse v1.3.0 h1:1Ep/RaYoSL1r5rTILHQQbyzHG8T4UP5ZbQTYTo4bdDc=
 github.com/maruel/panicparse v1.3.0/go.mod h1:vszMjr5QQ4F5FSRfraldcIA/BCw5xrdLL+zEcU2nRBs=
 github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
@@ -87,52 +114,81 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
-github.com/minio/sha256-simd v0.0.0-20190117184323-cc1980cb0338 h1:USW1+zAUkUSvk097CAX/i8KR3r6f+DHNhk6Xe025Oyw=
-github.com/minio/sha256-simd v0.0.0-20190117184323-cc1980cb0338/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc=
+github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M=
+github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/oschwald/geoip2-golang v1.3.0 h1:D+Hsdos1NARPbzZ2aInUHZL+dApIzo8E0ErJVsWcku8=
 github.com/oschwald/geoip2-golang v1.3.0/go.mod h1:0LTTzix/Ao1uMvOhAV4iLU0Lz7eCrP94qZWBTDKf0iE=
-github.com/oschwald/maxminddb-golang v0.0.0-20170901134056-26fe5ace1c70 h1:XGLYUmodtNzThosQ8GkMvj9TiIB/uWsP8NfxKSa3aDc=
-github.com/oschwald/maxminddb-golang v0.0.0-20170901134056-26fe5ace1c70/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
-github.com/petermattis/goid v0.0.0-20170816195418-3db12ebb2a59 h1:2pHcLyJYXivxVvpoCc29uo3GDU1qFfJ1ggXKGYMrM0E=
-github.com/petermattis/goid v0.0.0-20170816195418-3db12ebb2a59/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
+github.com/oschwald/maxminddb-golang v1.4.0 h1:5/rpmW41qrgSed4wK32rdznbkTSXHcraY2LOMJX4DMc=
+github.com/oschwald/maxminddb-golang v1.4.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
+github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ=
+github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.4 h1:Y8E/JaaPbmFSW2V81Ab/d8yZFYQQGbni1b1jPcG9Y6A=
-github.com/prometheus/client_golang v0.9.4/go.mod h1:oCXIBxdI62A4cR6aTRJCgetEjecSIYzOEaeAn4iYEpM=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8=
+github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI=
+github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo=
+github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9 h1:jmLW6izPBVlIbk4d+XgK9+sChGbVKxxOPmd9eqRHCjw=
-github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
+github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.0.4 h1:w8DjqFMJDjuVwdZBQoOozr4MVWOnwF7RcL/7uxBjY78=
+github.com/prometheus/procfs v0.0.4/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
+github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
+github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y=
 github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
+github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf h1:c9SV5NzG4KOk448TUE7iqCmb4E4y79CZF4zDdc1Jx3Q=
+github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
@@ -143,27 +199,37 @@ github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2 h1:6tuEEEpg+mxM82
 github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2/go.mod h1:Sn4ChoS7e4FxjCN1XHPVBT43AgnRLbuaB8pEc1Zcdjg=
 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
+github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs=
+github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
 github.com/thejerf/suture v3.0.2+incompatible h1:GtMydYcnK4zBJ0KL6Lx9vLzl6Oozb65wh252FTBxrvM=
 github.com/thejerf/suture v3.0.2+incompatible/go.mod h1:ibKwrVj+Uzf3XZdAiNWUouPaAbSoemxOHLmJmwheEMc=
 github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
+github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
+github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0 h1:okhMind4q9H1OxF44gNegWkiP4H/gsTFLalHFa4OOUI=
 github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0/go.mod h1:TTbGUfE+cXXceWtbTHq6lqcTvYPBKLNejBEbnUsQJtU=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM=
+golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -175,19 +241,29 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e h1:ZytStCyV048ZqDsWHiYDdoI2Vd4msMcrDECFxS+tL9c=
 golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
-golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/time v0.0.0-20170927054726-6dc17368e09b h1:3X+R0qq1+64izd8es+EttB6qcY+JDlVmAhpRXl7gpzU=
-golang.org/x/time v0.0.0-20170927054726-6dc17368e09b/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
-gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225 h1:JBwmEvLfCqgPcIq8MjVMQxsF3LVL4XG/HH0qiG0+IFY=
-gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
+gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
+gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

gui/black/assets/css/theme.css

@@ -24,6 +24,27 @@ a:hover,a:focus,a.focus{
     border-width: 2px !important;
 }
 
+
+.nav-tabs > li.active > a,
+.nav-tabs > li.active > a:hover,
+.nav-tabs > li.active > a:focus {
+  color: #3498db !important;
+  background-color: #222222 !important;
+  border: 1px solid #222222 !important;
+  border-bottom-color: transparent !important;
+  cursor: default;
+}
+
+.nav-tabs{
+    border-bottom: 1px solid #333;
+}
+
+.nav-tabs > li > a:hover,
+.nav-tabs > li > a:focus {
+  background-color: #222222 !important;
+  border: none !important;
+}
+
 .navbar-text, .dropdown>a, .dropdown-menu>li>a, .hidden-xs>a, .navbar-link {
     color: #aaa !important;
 }

gui/dark/assets/css/theme.css

@@ -28,6 +28,27 @@ a:hover,a:focus,a.focus{
 
 }
 
+.nav-tabs > li.active > a,
+.nav-tabs > li.active > a:hover,
+.nav-tabs > li.active > a:focus {
+  color: #3498db !important;
+  background-color: #424242 !important;
+  border: 1px solid #424242 !important;
+  border-bottom-color: transparent !important;
+  cursor: default;
+}
+
+.nav-tabs{
+    border-bottom: 1px solid #333;
+}
+
+.nav-tabs > li > a:hover,
+.nav-tabs > li > a:focus {
+  background-color: #424242 !important;
+  border: none !important;
+}
+
+
 .navbar-text, .dropdown>a, .dropdown-menu>li>a, .hidden-xs>a, .navbar-link {
     color: #aaa !important;
 }

gui/default/assets/css/overrides.css

@@ -246,6 +246,14 @@ a.toggler:hover {
     text-decoration: none;
 }
 
+/**
+ * Panel padding decrease
+ */
+
+.panel-collapse .panel-body {
+    padding: 5px;
+}
+
 /**
  * Progress bars with centered text
  */
@@ -348,6 +356,12 @@ ul.three-columns li, ul.two-columns li {
         * columns. */
         white-space: normal;
     }
+
+    .two-columns {
+        -webkit-column-count: 1;
+        -moz-column-count: 1;
+        column-count: 1;
+    }
 }
 
 @media (max-width:479px) {
@@ -392,7 +406,7 @@ ul.three-columns li, ul.two-columns li {
         max-width: 100%;
         width: 100%;
     }
-    
+
     /* all buttons, except panel headings, get bottom margin, as they won't fit
        beside each other anymore */
     .btn:not(.panel-heading),
@@ -400,4 +414,4 @@ ul.three-columns li, ul.two-columns li {
     .btn:not(.panel-heading) + .btn:not(.panel-heading) {
         margin-bottom: 1rem;
     }
-}
+}

gui/default/assets/css/theme.css

@@ -7,29 +7,5 @@
 
 */
 
-.panel-progress {
-    background: #3498db;
-}
-
-.identicon rect {
-    fill: #333;
-}
-
-.panel-warning .identicon rect {
-    fill: #fff;
-}
-
-.li-column {
-    background-color: rgb(236, 240, 241);
-    border-radius: 3px;
-}
-
-.panel-heading:hover, .panel-heading:focus {
-    text-decoration: none;
-}
-
-.fancytree-ext-filter-hide tr.fancytree-submatch span.fancytree-title,
-.fancytree-ext-filter-hide span.fancytree-node.fancytree-submatch span.fancytree-title {
-    color: black !important;
-    font-weight: lighter !important;
-}
+@import "../../theme-assets/dark/assets/css/theme.css" screen and (prefers-color-scheme: dark);
+@import "../../theme-assets/light/assets/css/theme.css" (prefers-color-scheme: light), (prefers-color-scheme: no-preference);

gui/default/index.html

@@ -42,7 +42,7 @@
       <p class="navbar-text hidden-xs" ng-class="{'hidden-sm':upgradeInfo && upgradeInfo.newer}">{{thisDeviceName()}}</p>
       <ul class="nav navbar-nav navbar-right">
         <li ng-if="upgradeInfo && upgradeInfo.newer" class="upgrade-newer">
-          <button type="button" class="btn navbar-btn btn-primary btn-sm" ng-click="upgrade()">
+          <button type="button" class="btn navbar-btn btn-primary btn-sm" data-toggle="modal" data-target="#upgrade">
             <span class="fas fa-arrow-circle-up"></span>
             <span class="hidden-xs" translate translate-value-version="{{upgradeInfo.latest}}">Upgrade To {%version%}</span>
           </button>
@@ -323,6 +323,11 @@
                     <span class="visible-xs" aria-label="{{'Scanning' | translate}}"><i class="fas fa-fw fa-search"></i></span>
                   </span>
                   <span ng-switch-when="idle"><span class="hidden-xs" translate>Up to Date</span><span class="visible-xs" aria-label="{{'Up to Date' | translate}}"><i class="fas fa-fw fa-check"></i></span></span>
+                  <span ng-switch-when="localadditions"><span class="hidden-xs" translate>Local Additions</span><span class="visible-xs" aria-label="{{'Local Additions' | translate}}"><i class="fas fa-fw fa-check"></i></span></span>
+                  <span ng-switch-when="sync-preparing">
+                    <span class="hidden-xs" translate>Preparing to Sync</span>
+                    <span class="visible-xs" aria-label="{{'Preparing to Sync' | translate}}"><i class="fas fa-fw fa-hourglass-half"></i></span>
+                  </span>
                   <span ng-switch-when="syncing">
                     <span class="hidden-xs" translate>Syncing</span>
                     <span>({{syncPercentage(folder.id) | percent}}, {{model[folder.id].needBytes | binary}}B)</span>
@@ -836,6 +841,7 @@
   <ng-include src="'syncthing/transfer/failedFilesModalView.html'"></ng-include>
   <ng-include src="'syncthing/transfer/remoteNeededFilesModalView.html'"></ng-include>
   <ng-include src="'syncthing/transfer/localChangedFilesModalView.html'"></ng-include>
+  <ng-include src="'syncthing/core/upgradeModalView.html'"></ng-include>
   <ng-include src="'syncthing/core/majorUpgradeModalView.html'"></ng-include>
   <ng-include src="'syncthing/core/aboutModalView.html'"></ng-include>
   <ng-include src="'syncthing/core/discoveryFailuresModalView.html'"></ng-include>

gui/default/syncthing/app.js

@@ -66,6 +66,14 @@ function folderCompare(a, b) {
     return labelA > labelB;
 }
 
+function deviceMap(l) {
+    var m = {};
+    l.forEach(function (r) {
+        m[r.deviceID] = r;
+    });
+    return m;
+}
+
 function folderMap(l) {
     var m = {};
     l.forEach(function (r) {

gui/default/syncthing/core/aboutModalView.html

@@ -1,7 +1,7 @@
 <modal id="about" status="info" icon="far fa-heart" heading="{{'About' | translate}}" large="yes" closeable="yes">
   <div class="modal-body">
     <h1 class="text-center">
-      <img alt="Syncthing" src="assets/img/logo-horizontal.svg" style="vertical-align: -16px" height="100" width="366" />
+      <img alt="Syncthing" src="assets/img/logo-horizontal.svg" style="max-width: 366px; vertical-align: -16px" />
       <br />
       <small>{{versionString()}}</small>
       <br />

gui/default/syncthing/core/logViewerModalView.html

@@ -8,7 +8,7 @@
 
       <div id="log-viewer-log" class="tab-pane in active">
         <label translate ng-if="logging.logEntries.length == 0">Loading...</label>
-        <textarea id="logViewerText" class="form-control" rows="20" ng-if="logging.logEntries.length != 0" readonly style="font-family: Consolas; font-size: 11px; overflow: auto;">{{ logging.content() }}</textarea>
+        <textarea id="logViewerText" class="form-control" rows="20" ng-if="logging.logEntries.length != 0" readonly style="font-family: Consolas, monospace; font-size: 11px; overflow: auto;">{{ logging.content() }}</textarea>
         <p translate class="help-block" ng-style="{'visibility': logging.paused ? 'visible' : 'hidden'}">Log tailing paused. Scroll to the bottom to continue.</p>
       </div>
 

gui/default/syncthing/core/syncthingController.js

@@ -60,7 +60,9 @@ angular.module('syncthing.core')
         } catch (exception) { }
 
         $scope.folderDefaults = {
+            sharedDevices: {},
             selectedDevices: {},
+            unrelatedDevices: {},
             type: "sendreceive",
             rescanIntervalS: 3600,
             fsWatcherDelayS: 10,
@@ -386,15 +388,8 @@ angular.module('syncthing.core')
                 });
             });
 
-            // If we're not listening on localhost, and there is no
-            // authentication configured, and the magic setting to silence the
-            // warning isn't set, then yell at the user.
-            var guiCfg = $scope.config.gui;
-            $scope.openNoAuth = guiCfg.address.substr(0, 4) !== "127."
-                && guiCfg.address.substr(0, 6) !== "[::1]:"
-                && (!guiCfg.user || !guiCfg.password)
-                && guiCfg.authMode !== 'ldap'
-                && !guiCfg.insecureAdminAccess;
+            refreshNoAuthWarning();
+            setDefaultTheme();
 
             if (!hasConfig) {
                 $scope.$emit('ConfigLoaded');
@@ -427,10 +422,33 @@ angular.module('syncthing.core')
                     }
                 }
                 $scope.discoveryFailed = discoveryFailed;
+
+                refreshNoAuthWarning();
+
                 console.log("refreshSystem", data);
             }).error($scope.emitHTTPError);
         }
 
+        function refreshNoAuthWarning() {
+            if (!$scope.system || !$scope.config) {
+                // We need both to be able to determine the state.
+                return
+            }
+
+            // If we're not listening on localhost, and there is no
+            // authentication configured, and the magic setting to silence the
+            // warning isn't set, then yell at the user.
+            var addr = $scope.system.guiAddressUsed;
+            var guiCfg = $scope.config.gui;
+            $scope.openNoAuth = addr.substr(0, 4) !== "127."
+                && addr.substr(0, 6) !== "[::1]:"
+                && addr.substr(0, 1) !== "/"
+                && (!guiCfg.user || !guiCfg.password)
+                && guiCfg.authMode !== 'ldap'
+                && !guiCfg.insecureAdminAccess;
+        }
+
+
         function refreshDiscoveryCache() {
             $http.get(urlbase + '/system/discovery').success(function (data) {
                 for (var device in data) {
@@ -634,6 +652,23 @@ angular.module('syncthing.core')
             $scope.remoteNeedDevice = undefined;
         }
 
+
+        function setDefaultTheme() {
+            if (!document.getElementById("fallback-theme-css")){
+
+                // check if no support for prefers-color-scheme
+                var colorSchemeNotSupported = typeof window.matchMedia === "undefined" || window.matchMedia('(prefers-color-scheme: dark)').media === 'not all';
+
+                if ($scope.config.gui.theme === "default" && colorSchemeNotSupported) {
+                    document.documentElement.style.display = 'none';
+                    document.head.insertAdjacentHTML(
+                      'beforeend',
+                      '<link id="fallback-theme-css" rel="stylesheet" href="theme-assets/light/assets/css/theme.css" onload="document.documentElement.style.display = \'\'">'
+                    );
+                }
+            }
+        }
+
         function saveIgnores(ignores, cb) {
             $http.post(urlbase + '/db/ignores?folder=' + encodeURIComponent($scope.currentFolder.id), {
                 ignore: ignores
@@ -753,25 +788,31 @@ angular.module('syncthing.core')
                 return 'paused';
             }
 
+            var folderInfo = $scope.model[folderCfg.id];
+
             // after restart syncthing process state may be empty
-            if (!$scope.model[folderCfg.id].state) {
+            if (!folderInfo.state) {
                 return 'unknown';
             }
 
-            var state = '' + $scope.model[folderCfg.id].state;
+            var state = '' + folderInfo.state;
             if (state === 'error') {
                 return 'stopped'; // legacy, the state is called "stopped" in the GUI
             }
-            if (state === 'idle' && $scope.model[folderCfg.id].needTotalItems > 0) {
+
+            if (state !== 'idle') {
+                return state;
+            }
+
+            if (folderInfo.needTotalItems > 0) {
                 return 'outofsync';
             }
             if ($scope.hasFailedFiles(folderCfg.id)) {
                 return 'faileditems';
             }
-            if (state === 'scanning') {
-                return state;
+            if (folderInfo.receiveOnlyTotalItems) {
+                return 'localadditions';
             }
-
             if (folderCfg.devices.length <= 1) {
                 return 'unshared';
             }
@@ -782,13 +823,13 @@ angular.module('syncthing.core')
         $scope.folderClass = function (folderCfg) {
             var status = $scope.folderStatus(folderCfg);
 
-            if (status === 'idle') {
+            if (status === 'idle' || status === 'localadditions') {
                 return 'success';
             }
             if (status == 'paused') {
                 return 'default';
             }
-            if (status === 'syncing' || status === 'scanning') {
+            if (status === 'syncing' || status === 'sync-preparing' || status === 'scanning') {
                 return 'primary';
             }
             if (status === 'unknown') {
@@ -838,6 +879,9 @@ angular.module('syncthing.core')
             // 32m 40s
             // 2h 32m
             // 4d 2h
+            // In case remaining scan time appears to be >31d, omit the
+            // details, i.e.:
+            // > 1 month
 
             if (!$scope.scanProgress[folder]) {
                 return "";
@@ -857,6 +901,9 @@ angular.module('syncthing.core')
             var res = [];
             if (seconds >= 86400) {
                 days = Math.floor(seconds / 86400);
+                if (days > 31) {
+                    return '> 1 month';
+                }
                 res.push('' + days + 'd')
                 seconds = seconds % 86400;
             }
@@ -947,6 +994,7 @@ angular.module('syncthing.core')
             for (var i = 0; i < folderListCache.length; i++) {
                 var status = $scope.folderStatus(folderListCache[i]);
                 switch (status) {
+                    case 'sync-preparing':
                     case 'syncing':
                         syncCount++;
                         break;
@@ -1279,6 +1327,13 @@ angular.module('syncthing.core')
                     $scope.protocolChanged = true;
                 }
 
+                // Parse strings to arrays before copying over
+                ['listenAddresses', 'globalAnnounceServers'].forEach(function (key) {
+                    $scope.tmpOptions[key] = $scope.tmpOptions["_" + key + "Str"].split(/[ ,]+/).map(function (x) {
+                        return x.trim();
+                    });
+                });
+
                 // Apply new settings locally
                 $scope.thisDeviceIn($scope.tmpDevices).name = $scope.tmpOptions.deviceName;
                 $scope.config.options = angular.copy($scope.tmpOptions);
@@ -1292,12 +1347,6 @@ angular.module('syncthing.core')
                 // here as well...
                 $scope.devices = $scope.config.devices;
 
-                ['listenAddresses', 'globalAnnounceServers'].forEach(function (key) {
-                    $scope.config.options[key] = $scope.config.options["_" + key + "Str"].split(/[ ,]+/).map(function (x) {
-                        return x.trim();
-                    });
-                });
-
                 $scope.saveConfig(function () {
                     if (themeChanged) {
                         document.location.reload(true);
@@ -1338,6 +1387,7 @@ angular.module('syncthing.core')
 
         $scope.upgrade = function () {
             restarting = true;
+            $('#upgrade').modal('hide');
             $('#majorUpgrade').modal('hide');
             $('#upgrading').modal();
             $http.post(urlbase + '/system/upgrade').success(function () {
@@ -1376,13 +1426,13 @@ angular.module('syncthing.core')
         };
 
         $scope.selectAllFolders = function () {
-            angular.forEach($scope.folders, function (id) {
+            angular.forEach($scope.folders, function (_, id) {
                 $scope.currentDevice.selectedFolders[id] = true;
             });
         };
 
         $scope.deSelectAllFolders = function () {
-            angular.forEach($scope.folders, function (id) {
+            angular.forEach($scope.folders, function (_, id) {
                 $scope.currentDevice.selectedFolders[id] = false;
             });
         };
@@ -1656,10 +1706,20 @@ angular.module('syncthing.core')
             if ($scope.currentFolder.path.length > 1 && $scope.currentFolder.path.slice(-1) === $scope.system.pathSeparator) {
                 $scope.currentFolder.path = $scope.currentFolder.path.slice(0, -1);
             }
+            // Cache complete device objects indexed by ID for lookups
+            var devMap = deviceMap($scope.devices)
+            $scope.currentFolder.sharedDevices = [];
             $scope.currentFolder.selectedDevices = {};
             $scope.currentFolder.devices.forEach(function (n) {
+                if (n.deviceID !== $scope.myID) {
+                    $scope.currentFolder.sharedDevices.push(devMap[n.deviceID]);
+                }
                 $scope.currentFolder.selectedDevices[n.deviceID] = true;
             });
+            $scope.currentFolder.unrelatedDevices = $scope.devices.filter(function (n) {
+                return n.deviceID !== $scope.myID
+                    && ! $scope.currentFolder.selectedDevices[n.deviceID]
+            });
             if ($scope.currentFolder.versioning && $scope.currentFolder.versioning.type === "trashcan") {
                 $scope.currentFolder.trashcanFileVersioning = true;
                 $scope.currentFolder.fileVersioningSelector = "trashcan";
@@ -1710,17 +1770,17 @@ angular.module('syncthing.core')
             $scope.editFolderModal();
         };
 
-        $scope.selectAllDevices = function () {
-            var devices = $scope.otherDevices();
+        $scope.selectAllSharedDevices = function (state) {
+            var devices = $scope.currentFolder.sharedDevices;
             for (var i = 0; i < devices.length; i++) {
-                $scope.currentFolder.selectedDevices[devices[i].deviceID] = true;
+                $scope.currentFolder.selectedDevices[devices[i].deviceID] = !!state;
             }
         };
 
-        $scope.deSelectAllDevices = function () {
-            var devices = $scope.otherDevices();
+        $scope.selectAllUnrelatedDevices = function (state) {
+            var devices = $scope.currentFolder.unrelatedDevices;
             for (var i = 0; i < devices.length; i++) {
-                $scope.currentFolder.selectedDevices[devices[i].deviceID] = false;
+                $scope.currentFolder.selectedDevices[devices[i].deviceID] = !!state;
             }
         };
 
@@ -1729,6 +1789,7 @@ angular.module('syncthing.core')
                 $scope.editingExisting = false;
                 $scope.currentFolder = angular.copy($scope.folderDefaults);
                 $scope.currentFolder.id = (data.random.substr(0, 5) + '-' + data.random.substr(5, 5)).toLowerCase();
+                $scope.currentFolder.unrelatedDevices = $scope.otherDevices();
                 $('#folder-ignores textarea').val("");
                 $('#folder-ignores textarea').removeAttr('disabled');
                 $scope.editFolderModal();
@@ -1744,6 +1805,7 @@ angular.module('syncthing.core')
                 importFromOtherDevice: true
             };
             $scope.currentFolder.selectedDevices[device] = true;
+            $scope.currentFolder.unrelatedDevices = $scope.otherDevices();
             $('#folder-ignores textarea').val("");
             $('#folder-ignores textarea').removeAttr('disabled');
             $scope.editFolderModal();
@@ -1769,7 +1831,9 @@ angular.module('syncthing.core')
                     });
                 }
             }
+            delete folderCfg.sharedDevices;
             delete folderCfg.selectedDevices;
+            delete folderCfg.unrelatedDevices;
 
             if (folderCfg.fileVersioningSelector === "trashcan") {
                 folderCfg.versioning = {
@@ -1992,6 +2056,9 @@ angular.module('syncthing.core')
                                     value.modTime = new Date(value.modTime);
                                     value.versionTime = new Date(value.versionTime);
                                 });
+                                values.sort(function (a, b) {
+                                    return b.versionTime - a.versionTime;
+                                });
                             });
                             if (closed) return;
                             $scope.restoreVersions.versions = data;

gui/default/syncthing/core/upgradeModalView.html

@@ -0,0 +1,18 @@
+<modal id="upgrade" status="warning" icon="fas fa-arrow-circle-up" heading="{{'Upgrade' | translate}}" large="no" closeable="yes">
+    <div class="modal-body">
+      <p>
+        <span translate>Are you sure you want to upgrade?</span>
+      </p>
+      <p>
+        <a ng-href="https://github.com/syncthing/syncthing/releases/tag/{{upgradeInfo.latest}}" target="_blank" translate>Release Notes</a>
+      </p>
+    </div>
+    <div class="modal-footer">
+      <button type="button" class="btn btn-primary btn-sm" ng-click="upgrade()">
+        <span class="fas fa-check"></span>&nbsp;<span translate>Upgrade</span>
+      </button>
+      <button type="button" class="btn btn-default btn-sm" data-dismiss="modal">
+        <span class="fas fa-times"></span>&nbsp;<span translate>Close</span>
+      </button>
+    </div>
+  </modal>

gui/default/syncthing/device/editDeviceModalView.html

@@ -15,10 +15,10 @@
               <datalist id="discovery-list">
                 <option ng-repeat="id in discovery" value="{{id}}" />
               </datalist>
-              <p class="help-block" ng-if="discovery">
+              <p class="help-block" ng-if="discovery && discovery.length !== 0">
                 <span translate>You can also select one of these nearby devices:</span>
                 <ul>
-                  <li ng-repeat="id in discovery"><a ng-click="currentDevice.deviceID = id">{{id}}</a></li>
+                  <li ng-repeat="id in discovery"><a href="#" ng-click="currentDevice.deviceID = id">{{id}}</a></li>
                 </ul>
               </p>
               <p class="help-block">

gui/default/syncthing/folder/editFolderModalView.html

@@ -44,15 +44,35 @@
           </div>
         </div>
         <div id="folder-sharing" class="tab-pane">
-          <div class="form-group">
-            <label translate for="devices">Share With Devices</label>
+          <div class="form-group" ng-if="currentFolder.sharedDevices.length">
+            <label translate>Currently Shared With Devices</label>
             <p class="help-block">
-              <span translate>Select the devices to share this folder with.</span>&emsp;
-              <small><a href="#" ng-click="selectAllDevices()" translate>Select All</a>&emsp;
-                <a href="#" ng-click="deSelectAllDevices()" translate>Deselect All</a></small>
+              <span translate>Deselect devices to stop sharing this folder with.</span>&emsp;
+              <small><a href="#" ng-click="selectAllSharedDevices(true)" translate>Select All</a>&emsp;
+                <a href="#" ng-click="selectAllSharedDevices(false)" translate>Deselect All</a></small>
             </p>
             <div class="row">
-              <div class="col-md-4" ng-repeat="device in otherDevices()">
+              <div class="col-md-4" ng-repeat="device in currentFolder.sharedDevices">
+                <div class="checkbox">
+                  <label>
+                    <input type="checkbox" ng-model="currentFolder.selectedDevices[device.deviceID]" /> {{deviceName(device)}}
+                  </label>
+                </div>
+              </div>
+            </div>
+          </div>
+          <div class="form-group" ng-if="currentFolder.unrelatedDevices.length || otherDevices().length <= 0">
+            <label translate>Unshared Devices</label>
+            <p class="help-block" ng-if="otherDevices().length > 0">
+              <span translate>Select additional devices to share this folder with.</span>&emsp;
+              <small><a href="#" ng-click="selectAllUnrelatedDevices(true)" translate>Select All</a>&emsp;
+                <a href="#" ng-click="selectAllUnrelatedDevices(false)" translate>Deselect All</a></small>
+            </p>
+            <p class="help-block" ng-if="otherDevices().length <= 0">
+              <span translate>There are no devices to share this folder with.</span>
+            </p>
+            <div class="row">
+              <div class="col-md-4" ng-repeat="device in currentFolder.unrelatedDevices">
                 <div class="checkbox">
                   <label>
                     <input type="checkbox" ng-model="currentFolder.selectedDevices[device.deviceID]" /> {{deviceName(device)}}

gui/light/assets/css/theme.css

@@ -0,0 +1,35 @@
+/*
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+*/
+
+.panel-progress {
+    background: #3498db;
+}
+
+.identicon rect {
+    fill: #333;
+}
+
+.panel-warning .identicon rect {
+    fill: #fff;
+}
+
+.li-column {
+    background-color: rgb(236, 240, 241);
+    border-radius: 3px;
+}
+
+.panel-heading:hover, .panel-heading:focus {
+    text-decoration: none;
+}
+
+.fancytree-ext-filter-hide tr.fancytree-submatch span.fancytree-title,
+.fancytree-ext-filter-hide span.fancytree-node.fancytree-submatch span.fancytree-title {
+    color: black !important;
+    font-weight: lighter !important;
+}

lib/api/api.go

@@ -8,11 +8,15 @@ package api
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"net"
 	"net/http"
 	"net/url"
@@ -55,10 +59,11 @@ import (
 var bcryptExpr = regexp.MustCompile(`^\$2[aby]\$\d+\$.{50,}`)
 
 const (
-	DefaultEventMask    = events.AllEvents &^ events.LocalChangeDetected &^ events.RemoteChangeDetected
-	DiskEventMask       = events.LocalChangeDetected | events.RemoteChangeDetected
-	EventSubBufferSize  = 1000
-	defaultEventTimeout = time.Minute
+	DefaultEventMask      = events.AllEvents &^ events.LocalChangeDetected &^ events.RemoteChangeDetected
+	DiskEventMask         = events.LocalChangeDetected | events.RemoteChangeDetected
+	EventSubBufferSize    = 1000
+	defaultEventTimeout   = time.Minute
+	httpsCertLifetimeDays = 820
 )
 
 type service struct {
@@ -70,6 +75,7 @@ type service struct {
 	model                model.Model
 	eventSubs            map[events.EventType]events.BufferedSubscription
 	eventSubsMut         sync.Mutex
+	evLogger             events.Logger
 	discoverer           discover.CachingMux
 	connectionsService   connections.Service
 	fss                  model.FolderSummaryService
@@ -79,11 +85,11 @@ type service struct {
 	contr                Controller
 	noUpgrade            bool
 	tlsDefaultCommonName string
-	stop                 chan struct{} // signals intentional stop
 	configChanged        chan struct{} // signals intentional listener close due to config change
 	started              chan string   // signals startup complete by sending the listener address, for testing only
 	startedOnce          chan struct{} // the service has started successfully at least once
 	startupErr           error
+	listenerAddr         net.Addr
 
 	guiErrors logger.Recorder
 	systemLog logger.Recorder
@@ -105,7 +111,7 @@ type Service interface {
 	WaitForStart() error
 }
 
-func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonName string, m model.Model, defaultSub, diskSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService connections.Service, urService *ur.Service, fss model.FolderSummaryService, errors, systemLog logger.Recorder, cpu Rater, contr Controller, noUpgrade bool) Service {
+func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonName string, m model.Model, defaultSub, diskSub events.BufferedSubscription, evLogger events.Logger, discoverer discover.CachingMux, connectionsService connections.Service, urService *ur.Service, fss model.FolderSummaryService, errors, systemLog logger.Recorder, cpu Rater, contr Controller, noUpgrade bool) Service {
 	s := &service{
 		id:      id,
 		cfg:     cfg,
@@ -116,6 +122,7 @@ func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonNam
 			DiskEventMask:    diskSub,
 		},
 		eventSubsMut:         sync.NewMutex(),
+		evLogger:             evLogger,
 		discoverer:           discoverer,
 		connectionsService:   connectionsService,
 		fss:                  fss,
@@ -130,7 +137,7 @@ func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonNam
 		configChanged:        make(chan struct{}),
 		startedOnce:          make(chan struct{}),
 	}
-	s.Service = util.AsService(s.serve)
+	s.Service = util.AsService(s.serve, s.String())
 	return s
 }
 
@@ -143,6 +150,12 @@ func (s *service) getListener(guiCfg config.GUIConfiguration) (net.Listener, err
 	httpsCertFile := locations.Get(locations.HTTPSCertFile)
 	httpsKeyFile := locations.Get(locations.HTTPSKeyFile)
 	cert, err := tls.LoadX509KeyPair(httpsCertFile, httpsKeyFile)
+
+	// If the certificate has expired or will expire in the next month, fail
+	// it and generate a new one.
+	if err == nil {
+		err = checkExpiry(cert)
+	}
 	if err != nil {
 		l.Infoln("Loading HTTPS certificate:", err)
 		l.Infoln("Creating new HTTPS certificate")
@@ -155,7 +168,7 @@ func (s *service) getListener(guiCfg config.GUIConfiguration) (net.Listener, err
 			name = s.tlsDefaultCommonName
 		}
 
-		cert, err = tlsutil.NewCertificate(httpsCertFile, httpsKeyFile, name)
+		cert, err = tlsutil.NewCertificate(httpsCertFile, httpsKeyFile, name, httpsCertLifetimeDays)
 	}
 	if err != nil {
 		return nil, err
@@ -195,7 +208,7 @@ func sendJSON(w http.ResponseWriter, jsonObject interface{}) {
 	fmt.Fprintf(w, "%s\n", bs)
 }
 
-func (s *service) serve(stop chan struct{}) {
+func (s *service) serve(ctx context.Context) {
 	listener, err := s.getListener(s.cfg.GUI())
 	if err != nil {
 		select {
@@ -220,6 +233,7 @@ func (s *service) serve(stop chan struct{}) {
 		return
 	}
 
+	s.listenerAddr = listener.Addr()
 	defer listener.Close()
 
 	s.cfg.Subscribe(s)
@@ -308,14 +322,14 @@ func (s *service) serve(stop chan struct{}) {
 
 	// Wrap everything in CSRF protection. The /rest prefix should be
 	// protected, other requests will grant cookies.
-	handler := csrfMiddleware(s.id.String()[:5], "/rest", guiCfg, mux)
+	var handler http.Handler = newCsrfManager(s.id.String()[:5], "/rest", guiCfg, mux, locations.Get(locations.CsrfTokens))
 
 	// Add our version and ID as a header to responses
 	handler = withDetailsMiddleware(s.id, handler)
 
 	// Wrap everything in basic auth, if user/password is set.
 	if guiCfg.IsAuthEnabled() {
-		handler = basicAuthAndSessionMiddleware("sessionid-"+s.id.String()[:5], guiCfg, s.cfg.LDAP(), handler)
+		handler = basicAuthAndSessionMiddleware("sessionid-"+s.id.String()[:5], guiCfg, s.cfg.LDAP(), handler, s.evLogger)
 	}
 
 	// Redirect to HTTPS if we are supposed to
@@ -338,6 +352,9 @@ func (s *service) serve(stop chan struct{}) {
 		// ReadTimeout must be longer than SyncthingController $scope.refresh
 		// interval to avoid HTTP keepalive/GUI refresh race.
 		ReadTimeout: 15 * time.Second,
+		// Prevent the HTTP server from logging stuff on its own. The things we
+		// care about we log ourselves from the handlers.
+		ErrorLog: log.New(ioutil.Discard, "", 0),
 	}
 
 	l.Infoln("GUI and API listening on", listener.Addr())
@@ -365,7 +382,7 @@ func (s *service) serve(stop chan struct{}) {
 	// Wait for stop, restart or error signals
 
 	select {
-	case <-stop:
+	case <-ctx.Done():
 		// Shutting down permanently
 		l.Debugln("shutting down (stop)")
 	case <-s.configChanged:
@@ -375,6 +392,7 @@ func (s *service) serve(stop chan struct{}) {
 		// Restart due to listen/serve failure
 		l.Warnln("GUI/API:", err, "(restarting)")
 	}
+	srv.Close()
 }
 
 // Complete implements suture.IsCompletable, which signifies to the supervisor
@@ -751,11 +769,21 @@ func (s *service) getSystemConnections(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *service) getDeviceStats(w http.ResponseWriter, r *http.Request) {
-	sendJSON(w, s.model.DeviceStatistics())
+	stats, err := s.model.DeviceStatistics()
+	if err != nil {
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+	sendJSON(w, stats)
 }
 
 func (s *service) getFolderStats(w http.ResponseWriter, r *http.Request) {
-	sendJSON(w, s.model.FolderStatistics())
+	stats, err := s.model.FolderStatistics()
+	if err != nil {
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+	sendJSON(w, stats)
 }
 
 func (s *service) getDBFile(w http.ResponseWriter, r *http.Request) {
@@ -907,6 +935,7 @@ func (s *service) getSystemStatus(w http.ResponseWriter, r *http.Request) {
 	res["uptime"] = s.urService.UptimeS()
 	res["startTime"] = ur.StartTime
 	res["guiAddressOverridden"] = s.cfg.GUI().IsOverridden()
+	res["guiAddressUsed"] = s.listenerAddr.String()
 
 	sendJSON(w, res)
 }
@@ -1211,7 +1240,7 @@ func (s *service) getEventSub(mask events.EventType) events.BufferedSubscription
 	s.eventSubsMut.Lock()
 	bufsub, ok := s.eventSubs[mask]
 	if !ok {
-		evsub := events.Default.Subscribe(mask)
+		evsub := s.evLogger.Subscribe(mask)
 		bufsub = events.NewBufferedSubscription(evsub, EventSubBufferSize)
 		s.eventSubs[mask] = bufsub
 	}
@@ -1564,10 +1593,10 @@ func (s *service) getHeapProf(w http.ResponseWriter, r *http.Request) {
 	pprof.WriteHeapProfile(w)
 }
 
-func toJsonFileInfoSlice(fs []db.FileInfoTruncated) []jsonDBFileInfo {
-	res := make([]jsonDBFileInfo, len(fs))
+func toJsonFileInfoSlice(fs []db.FileInfoTruncated) []jsonFileInfoTrunc {
+	res := make([]jsonFileInfoTrunc, len(fs))
 	for i, f := range fs {
-		res[i] = jsonDBFileInfo(f)
+		res[i] = jsonFileInfoTrunc(f)
 	}
 	return res
 }
@@ -1577,45 +1606,39 @@ func toJsonFileInfoSlice(fs []db.FileInfoTruncated) []jsonDBFileInfo {
 type jsonFileInfo protocol.FileInfo
 
 func (f jsonFileInfo) MarshalJSON() ([]byte, error) {
-	return json.Marshal(map[string]interface{}{
-		"name":          f.Name,
-		"type":          f.Type,
-		"size":          f.Size,
-		"permissions":   fmt.Sprintf("%#o", f.Permissions),
-		"deleted":       f.Deleted,
-		"invalid":       protocol.FileInfo(f).IsInvalid(),
-		"ignored":       protocol.FileInfo(f).IsIgnored(),
-		"mustRescan":    protocol.FileInfo(f).MustRescan(),
-		"noPermissions": f.NoPermissions,
-		"modified":      protocol.FileInfo(f).ModTime(),
-		"modifiedBy":    f.ModifiedBy.String(),
-		"sequence":      f.Sequence,
-		"numBlocks":     len(f.Blocks),
-		"version":       jsonVersionVector(f.Version),
-		"localFlags":    f.LocalFlags,
-	})
+	m := fileIntfJSONMap(protocol.FileInfo(f))
+	m["numBlocks"] = len(f.Blocks)
+	return json.Marshal(m)
 }
 
-type jsonDBFileInfo db.FileInfoTruncated
+type jsonFileInfoTrunc db.FileInfoTruncated
 
-func (f jsonDBFileInfo) MarshalJSON() ([]byte, error) {
-	return json.Marshal(map[string]interface{}{
-		"name":          f.Name,
-		"type":          f.Type.String(),
-		"size":          f.Size,
-		"permissions":   fmt.Sprintf("%#o", f.Permissions),
-		"deleted":       f.Deleted,
-		"invalid":       db.FileInfoTruncated(f).IsInvalid(),
-		"ignored":       db.FileInfoTruncated(f).IsIgnored(),
-		"mustRescan":    db.FileInfoTruncated(f).MustRescan(),
-		"noPermissions": f.NoPermissions,
-		"modified":      db.FileInfoTruncated(f).ModTime(),
-		"modifiedBy":    f.ModifiedBy.String(),
-		"sequence":      f.Sequence,
-		"numBlocks":     nil, // explicitly unknown
-		"version":       jsonVersionVector(f.Version),
-		"localFlags":    f.LocalFlags,
-	})
+func (f jsonFileInfoTrunc) MarshalJSON() ([]byte, error) {
+	m := fileIntfJSONMap(db.FileInfoTruncated(f))
+	m["numBlocks"] = nil // explicitly unknown
+	return json.Marshal(m)
+}
+
+func fileIntfJSONMap(f db.FileIntf) map[string]interface{} {
+	out := map[string]interface{}{
+		"name":          f.FileName(),
+		"type":          f.FileType().String(),
+		"size":          f.FileSize(),
+		"deleted":       f.IsDeleted(),
+		"invalid":       f.IsInvalid(),
+		"ignored":       f.IsIgnored(),
+		"mustRescan":    f.MustRescan(),
+		"noPermissions": !f.HasPermissionBits(),
+		"modified":      f.ModTime(),
+		"modifiedBy":    f.FileModifiedBy().String(),
+		"sequence":      f.SequenceNo(),
+		"version":       jsonVersionVector(f.FileVersion()),
+		"localFlags":    f.FileLocalFlags(),
+	}
+	if f.HasPermissionBits() {
+		out["permissions"] = fmt.Sprintf("%#o", f.FilePermissions())
+	}
+	return out
 }
 
 type jsonVersionVector protocol.Vector
@@ -1669,3 +1692,45 @@ func addressIsLocalhost(addr string) bool {
 		return ip.IsLoopback()
 	}
 }
+
+func checkExpiry(cert tls.Certificate) error {
+	leaf := cert.Leaf
+	if leaf == nil {
+		// Leaf can be nil or not, depending on how parsed the certificate
+		// was when we got it.
+		if len(cert.Certificate) < 1 {
+			// can't happen
+			return errors.New("no certificate in certificate")
+		}
+		var err error
+		leaf, err = x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return err
+		}
+	}
+
+	if leaf.Subject.String() != leaf.Issuer.String() ||
+		len(leaf.DNSNames) != 0 || len(leaf.IPAddresses) != 0 {
+		// The certificate is not self signed, or has DNS/IP attributes we don't
+		// add, so we leave it alone.
+		return nil
+	}
+
+	if leaf.NotAfter.Before(time.Now()) {
+		return errors.New("certificate has expired")
+	}
+	if leaf.NotAfter.Before(time.Now().Add(30 * 24 * time.Hour)) {
+		return errors.New("certificate will soon expire")
+	}
+
+	// On macOS, check for certificates issued on or after July 1st, 2019,
+	// with a longer validity time than 825 days.
+	cutoff := time.Date(2019, 7, 1, 0, 0, 0, 0, time.UTC)
+	if runtime.GOOS == "darwin" &&
+		leaf.NotBefore.After(cutoff) &&
+		leaf.NotAfter.Sub(leaf.NotBefore) > 825*24*time.Hour {
+		return errors.New("certificate incompatible with macOS 10.15 (Catalina)")
+	}
+
+	return nil
+}

lib/api/api_auth.go

@@ -28,14 +28,14 @@ var (
 	sessionsMut = sync.NewMutex()
 )
 
-func emitLoginAttempt(success bool, username string) {
-	events.Default.Log(events.LoginAttempt, map[string]interface{}{
+func emitLoginAttempt(success bool, username string, evLogger events.Logger) {
+	evLogger.Log(events.LoginAttempt, map[string]interface{}{
 		"success":  success,
 		"username": username,
 	})
 }
 
-func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfiguration, ldapCfg config.LDAPConfiguration, next http.Handler) http.Handler {
+func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfiguration, ldapCfg config.LDAPConfiguration, next http.Handler, evLogger events.Logger) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if guiCfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
 			next.ServeHTTP(w, r)
@@ -94,7 +94,7 @@ func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfigura
 		}
 
 		if !authOk {
-			emitLoginAttempt(false, username)
+			emitLoginAttempt(false, username, evLogger)
 			error()
 			return
 		}
@@ -109,7 +109,7 @@ func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfigura
 			MaxAge: 0,
 		})
 
-		emitLoginAttempt(true, username)
+		emitLoginAttempt(true, username, evLogger)
 		next.ServeHTTP(w, r)
 	})
 }

lib/api/api_auth_test.go

@@ -19,6 +19,8 @@ func init() {
 }
 
 func TestStaticAuthOK(t *testing.T) {
+	t.Parallel()
+
 	ok := authStatic("user", "pass", "user", string(passwordHashBytes))
 	if !ok {
 		t.Fatalf("should pass auth")
@@ -26,6 +28,8 @@ func TestStaticAuthOK(t *testing.T) {
 }
 
 func TestSimpleAuthUsernameFail(t *testing.T) {
+	t.Parallel()
+
 	ok := authStatic("userWRONG", "pass", "user", string(passwordHashBytes))
 	if ok {
 		t.Fatalf("should fail auth")
@@ -33,6 +37,8 @@ func TestSimpleAuthUsernameFail(t *testing.T) {
 }
 
 func TestStaticAuthPasswordFail(t *testing.T) {
+	t.Parallel()
+
 	ok := authStatic("user", "passWRONG", "user", string(passwordHashBytes))
 	if ok {
 		t.Fatalf("should fail auth")

lib/api/api_csrf.go

@@ -13,83 +13,103 @@ import (
 	"os"
 	"strings"
 
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
-// csrfTokens is a list of valid tokens. It is sorted so that the most
-// recently used token is first in the list. New tokens are added to the front
-// of the list (as it is the most recently used at that time). The list is
-// pruned to a maximum of maxCsrfTokens, throwing away the least recently used
-// tokens.
-var csrfTokens []string
-var csrfMut = sync.NewMutex()
-
 const maxCsrfTokens = 25
 
+type csrfManager struct {
+	// tokens is a list of valid tokens. It is sorted so that the most
+	// recently used token is first in the list. New tokens are added to the front
+	// of the list (as it is the most recently used at that time). The list is
+	// pruned to a maximum of maxCsrfTokens, throwing away the least recently used
+	// tokens.
+	tokens    []string
+	tokensMut sync.Mutex
+
+	unique          string
+	prefix          string
+	apiKeyValidator apiKeyValidator
+	next            http.Handler
+	saveLocation    string
+}
+
+type apiKeyValidator interface {
+	IsValidAPIKey(key string) bool
+}
+
 // Check for CSRF token on /rest/ URLs. If a correct one is not given, reject
 // the request with 403. For / and /index.html, set a new CSRF cookie if none
 // is currently set.
-func csrfMiddleware(unique string, prefix string, cfg config.GUIConfiguration, next http.Handler) http.Handler {
-	loadCsrfTokens()
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// Allow requests carrying a valid API key
-		if cfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
-			// Set the access-control-allow-origin header for CORS requests
-			// since a valid API key has been provided
-			w.Header().Add("Access-Control-Allow-Origin", "*")
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		if strings.HasPrefix(r.URL.Path, "/rest/debug") {
-			// Debugging functions are only available when explicitly
-			// enabled, and can be accessed without a CSRF token
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		// Allow requests for anything not under the protected path prefix,
-		// and set a CSRF cookie if there isn't already a valid one.
-		if !strings.HasPrefix(r.URL.Path, prefix) {
-			cookie, err := r.Cookie("CSRF-Token-" + unique)
-			if err != nil || !validCsrfToken(cookie.Value) {
-				l.Debugln("new CSRF cookie in response to request for", r.URL)
-				cookie = &http.Cookie{
-					Name:  "CSRF-Token-" + unique,
-					Value: newCsrfToken(),
-				}
-				http.SetCookie(w, cookie)
-			}
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		// Verify the CSRF token
-		token := r.Header.Get("X-CSRF-Token-" + unique)
-		if !validCsrfToken(token) {
-			http.Error(w, "CSRF Error", 403)
-			return
-		}
-
-		next.ServeHTTP(w, r)
-	})
+func newCsrfManager(unique string, prefix string, apiKeyValidator apiKeyValidator, next http.Handler, saveLocation string) *csrfManager {
+	m := &csrfManager{
+		tokensMut:       sync.NewMutex(),
+		unique:          unique,
+		prefix:          prefix,
+		apiKeyValidator: apiKeyValidator,
+		next:            next,
+		saveLocation:    saveLocation,
+	}
+	m.load()
+	return m
 }
 
-func validCsrfToken(token string) bool {
-	csrfMut.Lock()
-	defer csrfMut.Unlock()
-	for i, t := range csrfTokens {
+func (m *csrfManager) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Allow requests carrying a valid API key
+	if m.apiKeyValidator.IsValidAPIKey(r.Header.Get("X-API-Key")) {
+		// Set the access-control-allow-origin header for CORS requests
+		// since a valid API key has been provided
+		w.Header().Add("Access-Control-Allow-Origin", "*")
+		m.next.ServeHTTP(w, r)
+		return
+	}
+
+	if strings.HasPrefix(r.URL.Path, "/rest/debug") {
+		// Debugging functions are only available when explicitly
+		// enabled, and can be accessed without a CSRF token
+		m.next.ServeHTTP(w, r)
+		return
+	}
+
+	// Allow requests for anything not under the protected path prefix,
+	// and set a CSRF cookie if there isn't already a valid one.
+	if !strings.HasPrefix(r.URL.Path, m.prefix) {
+		cookie, err := r.Cookie("CSRF-Token-" + m.unique)
+		if err != nil || !m.validToken(cookie.Value) {
+			l.Debugln("new CSRF cookie in response to request for", r.URL)
+			cookie = &http.Cookie{
+				Name:  "CSRF-Token-" + m.unique,
+				Value: m.newToken(),
+			}
+			http.SetCookie(w, cookie)
+		}
+		m.next.ServeHTTP(w, r)
+		return
+	}
+
+	// Verify the CSRF token
+	token := r.Header.Get("X-CSRF-Token-" + m.unique)
+	if !m.validToken(token) {
+		http.Error(w, "CSRF Error", http.StatusForbidden)
+		return
+	}
+
+	m.next.ServeHTTP(w, r)
+}
+
+func (m *csrfManager) validToken(token string) bool {
+	m.tokensMut.Lock()
+	defer m.tokensMut.Unlock()
+	for i, t := range m.tokens {
 		if t == token {
 			if i > 0 {
 				// Move this token to the head of the list. Copy the tokens at
 				// the front one step to the right and then replace the token
 				// at the head.
-				copy(csrfTokens[1:], csrfTokens[:i+1])
-				csrfTokens[0] = token
+				copy(m.tokens[1:], m.tokens[:i+1])
+				m.tokens[0] = token
 			}
 			return true
 		}
@@ -97,40 +117,47 @@ func validCsrfToken(token string) bool {
 	return false
 }
 
-func newCsrfToken() string {
+func (m *csrfManager) newToken() string {
 	token := rand.String(32)
 
-	csrfMut.Lock()
-	csrfTokens = append([]string{token}, csrfTokens...)
-	if len(csrfTokens) > maxCsrfTokens {
-		csrfTokens = csrfTokens[:maxCsrfTokens]
+	m.tokensMut.Lock()
+	m.tokens = append([]string{token}, m.tokens...)
+	if len(m.tokens) > maxCsrfTokens {
+		m.tokens = m.tokens[:maxCsrfTokens]
 	}
-	defer csrfMut.Unlock()
+	defer m.tokensMut.Unlock()
 
-	saveCsrfTokens()
+	m.save()
 
 	return token
 }
 
-func saveCsrfTokens() {
+func (m *csrfManager) save() {
 	// We're ignoring errors in here. It's not super critical and there's
 	// nothing relevant we can do about them anyway...
 
-	name := locations.Get(locations.CsrfTokens)
-	f, err := osutil.CreateAtomic(name)
+	if m.saveLocation == "" {
+		return
+	}
+
+	f, err := osutil.CreateAtomic(m.saveLocation)
 	if err != nil {
 		return
 	}
 
-	for _, t := range csrfTokens {
+	for _, t := range m.tokens {
 		fmt.Fprintln(f, t)
 	}
 
 	f.Close()
 }
 
-func loadCsrfTokens() {
-	f, err := os.Open(locations.Get(locations.CsrfTokens))
+func (m *csrfManager) load() {
+	if m.saveLocation == "" {
+		return
+	}
+
+	f, err := os.Open(m.saveLocation)
 	if err != nil {
 		return
 	}
@@ -138,6 +165,6 @@ func loadCsrfTokens() {
 
 	s := bufio.NewScanner(f)
 	for s.Scan() {
-		csrfTokens = append(csrfTokens, s.Text())
+		m.tokens = append(m.tokens, s.Text())
 	}
 }

lib/api/api_statics.go

@@ -23,21 +23,25 @@ import (
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
+const themePrefix = "theme-assets/"
+
 type staticsServer struct {
 	assetDir        string
 	assets          map[string][]byte
 	availableThemes []string
 
-	mut   sync.RWMutex
-	theme string
+	mut             sync.RWMutex
+	theme           string
+	lastThemeChange time.Time
 }
 
 func newStaticsServer(theme, assetDir string) *staticsServer {
 	s := &staticsServer{
-		assetDir: assetDir,
-		assets:   auto.Assets(),
-		mut:      sync.NewRWMutex(),
-		theme:    theme,
+		assetDir:        assetDir,
+		assets:          auto.Assets(),
+		mut:             sync.NewRWMutex(),
+		theme:           theme,
+		lastThemeChange: time.Now().UTC(),
 	}
 
 	seen := make(map[string]struct{})
@@ -86,8 +90,23 @@ func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) {
 
 	s.mut.RLock()
 	theme := s.theme
+	modificationTime := s.lastThemeChange
 	s.mut.RUnlock()
 
+	// If path starts with special prefix, get theme and file from path
+	if strings.HasPrefix(file, themePrefix) {
+		path := file[len(themePrefix):]
+		i := strings.IndexRune(path, '/')
+
+		if i == -1 {
+			http.NotFound(w, r)
+			return
+		}
+
+		theme = path[:i]
+		file = path[i+1:]
+	}
+
 	// Check for an override for the current theme.
 	if s.assetDir != "" {
 		p := filepath.Join(s.assetDir, theme, filepath.FromSlash(file))
@@ -125,14 +144,12 @@ func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	etag := fmt.Sprintf("%d", auto.Generated)
-	modified := time.Unix(auto.Generated, 0).UTC()
-
-	w.Header().Set("Last-Modified", modified.Format(http.TimeFormat))
+	etag := fmt.Sprintf("%d", modificationTime.Unix())
+	w.Header().Set("Last-Modified", modificationTime.Format(http.TimeFormat))
 	w.Header().Set("Etag", etag)
 
 	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil {
-		if modified.Equal(t) || modified.Before(t) {
+		if modificationTime.Equal(t) || modificationTime.Before(t) {
 			w.WriteHeader(http.StatusNotModified)
 			return
 		}
@@ -199,6 +216,7 @@ func (s *staticsServer) mimeTypeForFile(file string) string {
 func (s *staticsServer) setTheme(theme string) {
 	s.mut.Lock()
 	s.theme = theme
+	s.lastThemeChange = time.Now().UTC()
 	s.mut.Unlock()
 }
 

lib/api/api_test.go

@@ -18,6 +18,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"testing"
@@ -31,6 +32,7 @@ import (
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/ur"
 	"github.com/thejerf/suture"
 )
@@ -52,7 +54,7 @@ func TestMain(m *testing.M) {
 }
 
 func TestCSRFToken(t *testing.T) {
-	defer os.Remove(token)
+	t.Parallel()
 
 	max := 250
 	int := 5
@@ -61,11 +63,13 @@ func TestCSRFToken(t *testing.T) {
 		int = 2
 	}
 
-	t1 := newCsrfToken()
-	t2 := newCsrfToken()
+	m := newCsrfManager("unique", "prefix", config.GUIConfiguration{}, nil, "")
 
-	t3 := newCsrfToken()
-	if !validCsrfToken(t3) {
+	t1 := m.newToken()
+	t2 := m.newToken()
+
+	t3 := m.newToken()
+	if !m.validToken(t3) {
 		t.Fatal("t3 should be valid")
 	}
 
@@ -73,36 +77,38 @@ func TestCSRFToken(t *testing.T) {
 		if i%int == 0 {
 			// t1 and t2 should remain valid by virtue of us checking them now
 			// and then.
-			if !validCsrfToken(t1) {
+			if !m.validToken(t1) {
 				t.Fatal("t1 should be valid at iteration", i)
 			}
-			if !validCsrfToken(t2) {
+			if !m.validToken(t2) {
 				t.Fatal("t2 should be valid at iteration", i)
 			}
 		}
 
 		// The newly generated token is always valid
-		t4 := newCsrfToken()
-		if !validCsrfToken(t4) {
+		t4 := m.newToken()
+		if !m.validToken(t4) {
 			t.Fatal("t4 should be valid at iteration", i)
 		}
 	}
 
-	if validCsrfToken(t3) {
+	if m.validToken(t3) {
 		t.Fatal("t3 should have expired by now")
 	}
 }
 
 func TestStopAfterBrokenConfig(t *testing.T) {
+	t.Parallel()
+
 	cfg := config.Configuration{
 		GUI: config.GUIConfiguration{
 			RawAddress: "127.0.0.1:0",
 			RawUseTLS:  false,
 		},
 	}
-	w := config.Wrap("/dev/null", cfg)
+	w := config.Wrap("/dev/null", cfg, events.NoopLogger)
 
-	srv := New(protocol.LocalDeviceID, w, "", "syncthing", nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, false).(*service)
+	srv := New(protocol.LocalDeviceID, w, "", "syncthing", nil, nil, nil, events.NoopLogger, nil, nil, nil, nil, nil, nil, nil, nil, false).(*service)
 	defer os.Remove(token)
 	srv.started = make(chan string)
 
@@ -134,6 +140,8 @@ func TestStopAfterBrokenConfig(t *testing.T) {
 }
 
 func TestAssetsDir(t *testing.T) {
+	t.Parallel()
+
 	// For any given request to $FILE, we should return the first found of
 	//  - assetsdir/$THEME/$FILE
 	//  - compiled in asset $THEME/$FILE
@@ -208,6 +216,8 @@ func expectURLToContain(t *testing.T, url, exp string) {
 }
 
 func TestDirNames(t *testing.T) {
+	t.Parallel()
+
 	names := dirNames("testdata")
 	expected := []string{"config", "default", "foo", "testfolder"}
 	if diff, equal := messagediff.PrettyDiff(expected, names); !equal {
@@ -224,6 +234,8 @@ type httpTestCase struct {
 }
 
 func TestAPIServiceRequests(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -434,6 +446,8 @@ func testHTTPRequest(t *testing.T, baseURL string, tc httpTestCase, apikey strin
 }
 
 func TestHTTPLogin(t *testing.T) {
+	t.Parallel()
+
 	cfg := new(mockedConfig)
 	cfg.gui.User = "üser"
 	cfg.gui.Password = "$2a$10$IdIZTxTg/dCNuNEGlmLynOjqg4B1FvDKuIV5e0BB3pnWVHNb8.GSq" // bcrypt of "räksmörgås" in UTF-8
@@ -512,8 +526,8 @@ func startHTTP(cfg *mockedConfig) (string, error) {
 
 	// Instantiate the API service
 	urService := ur.New(cfg, m, connections, false)
-	summaryService := model.NewFolderSummaryService(cfg, m, protocol.LocalDeviceID)
-	svc := New(protocol.LocalDeviceID, cfg, assetDir, "syncthing", m, eventSub, diskEventSub, discoverer, connections, urService, summaryService, errorLog, systemLog, cpu, nil, false).(*service)
+	summaryService := model.NewFolderSummaryService(cfg, m, protocol.LocalDeviceID, events.NoopLogger)
+	svc := New(protocol.LocalDeviceID, cfg, assetDir, "syncthing", m, eventSub, diskEventSub, events.NoopLogger, discoverer, connections, urService, summaryService, errorLog, systemLog, cpu, nil, false).(*service)
 	defer os.Remove(token)
 	svc.started = addrChan
 
@@ -541,6 +555,8 @@ func startHTTP(cfg *mockedConfig) (string, error) {
 }
 
 func TestCSRFRequired(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -550,7 +566,7 @@ func TestCSRFRequired(t *testing.T) {
 	}
 
 	cli := &http.Client{
-		Timeout: time.Second,
+		Timeout: time.Minute,
 	}
 
 	// Getting the base URL (i.e. "/") should succeed.
@@ -614,6 +630,8 @@ func TestCSRFRequired(t *testing.T) {
 }
 
 func TestRandomString(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -663,6 +681,8 @@ func TestRandomString(t *testing.T) {
 }
 
 func TestConfigPostOK(t *testing.T) {
+	t.Parallel()
+
 	cfg := bytes.NewBuffer([]byte(`{
 		"version": 15,
 		"folders": [
@@ -684,6 +704,8 @@ func TestConfigPostOK(t *testing.T) {
 }
 
 func TestConfigPostDupFolder(t *testing.T) {
+	t.Parallel()
+
 	cfg := bytes.NewBuffer([]byte(`{
 		"version": 15,
 		"folders": [
@@ -719,6 +741,8 @@ func testConfigPost(data io.Reader) (*http.Response, error) {
 }
 
 func TestHostCheck(t *testing.T) {
+	t.Parallel()
+
 	// An API service bound to localhost should reject non-localhost host Headers
 
 	cfg := new(mockedConfig)
@@ -826,6 +850,11 @@ func TestHostCheck(t *testing.T) {
 
 	// This should all work over IPv6 as well
 
+	if runningInContainer() {
+		// Working IPv6 in Docker can't be taken for granted.
+		return
+	}
+
 	cfg = new(mockedConfig)
 	cfg.gui.RawAddress = "[::1]:0"
 	baseURL, err = startHTTP(cfg)
@@ -872,6 +901,8 @@ func TestHostCheck(t *testing.T) {
 }
 
 func TestAddressIsLocalhost(t *testing.T) {
+	t.Parallel()
+
 	testcases := []struct {
 		address string
 		result  bool
@@ -915,6 +946,8 @@ func TestAddressIsLocalhost(t *testing.T) {
 }
 
 func TestAccessControlAllowOriginHeader(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -943,6 +976,8 @@ func TestAccessControlAllowOriginHeader(t *testing.T) {
 }
 
 func TestOptionsRequest(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -976,10 +1011,12 @@ func TestOptionsRequest(t *testing.T) {
 }
 
 func TestEventMasks(t *testing.T) {
+	t.Parallel()
+
 	cfg := new(mockedConfig)
 	defSub := new(mockedEventSub)
 	diskSub := new(mockedEventSub)
-	svc := New(protocol.LocalDeviceID, cfg, "", "syncthing", nil, defSub, diskSub, nil, nil, nil, nil, nil, nil, nil, nil, false).(*service)
+	svc := New(protocol.LocalDeviceID, cfg, "", "syncthing", nil, defSub, diskSub, events.NoopLogger, nil, nil, nil, nil, nil, nil, nil, nil, false).(*service)
 	defer os.Remove(token)
 
 	if mask := svc.getEventMask(""); mask != DefaultEventMask {
@@ -1008,6 +1045,8 @@ func TestEventMasks(t *testing.T) {
 }
 
 func TestBrowse(t *testing.T) {
+	t.Parallel()
+
 	pathSep := string(os.PathSeparator)
 
 	tmpDir, err := ioutil.TempDir("", "syncthing")
@@ -1060,6 +1099,8 @@ func TestBrowse(t *testing.T) {
 }
 
 func TestPrefixMatch(t *testing.T) {
+	t.Parallel()
+
 	cases := []struct {
 		s        string
 		prefix   string
@@ -1079,6 +1120,44 @@ func TestPrefixMatch(t *testing.T) {
 	}
 }
 
+func TestCheckExpiry(t *testing.T) {
+	dir, err := ioutil.TempDir("", "syncthing-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	// Self signed certificates expiring in less than a month are errored so we
+	// can regenerate in time.
+	crt, err := tlsutil.NewCertificate(filepath.Join(dir, "crt"), filepath.Join(dir, "key"), "foo.example.com", 29)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := checkExpiry(crt); err == nil {
+		t.Error("expected expiry error")
+	}
+
+	// Certificates with at least 31 days of life left are fine.
+	crt, err = tlsutil.NewCertificate(filepath.Join(dir, "crt"), filepath.Join(dir, "key"), "foo.example.com", 31)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := checkExpiry(crt); err != nil {
+		t.Error("expected no error:", err)
+	}
+
+	if runtime.GOOS == "darwin" {
+		// Certificates with too long an expiry time are not allowed on macOS
+		crt, err = tlsutil.NewCertificate(filepath.Join(dir, "crt"), filepath.Join(dir, "key"), "foo.example.com", 1000)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := checkExpiry(crt); err == nil {
+			t.Error("expected expiry error")
+		}
+	}
+}
+
 func equalStrings(a, b []string) bool {
 	if len(a) != len(b) {
 		return false
@@ -1090,3 +1169,24 @@ func equalStrings(a, b []string) bool {
 	}
 	return true
 }
+
+// runningInContainer returns true if we are inside Docker or LXC. It might
+// be prone to false negatives if things change in the future, but likely
+// not false positives.
+func runningInContainer() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+
+	bs, err := ioutil.ReadFile("/proc/1/cgroup")
+	if err != nil {
+		return false
+	}
+	if bytes.Contains(bs, []byte("/docker/")) {
+		return true
+	}
+	if bytes.Contains(bs, []byte("/lxc/")) {
+		return true
+	}
+	return false
+}

lib/api/debug.go

@@ -7,9 +7,6 @@
 package api
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
@@ -24,5 +21,7 @@ func shouldDebugHTTP() bool {
 func init() {
 	// The debug facility was originally named "http", changed in:
 	// https://github.com/syncthing/syncthing/pull/5548
-	l.SetDebug("api", strings.Contains(os.Getenv("STTRACE"), "api") || strings.Contains(os.Getenv("STTRACE"), "http") || os.Getenv("STTRACE") == "all")
+	if l.IsTraced("http") {
+		l.SetDebug("api", true)
+	}
 }

lib/api/mocked_connections_test.go

@@ -27,3 +27,7 @@ func (m *mockedConnections) NATType() string {
 func (m *mockedConnections) Serve() {}
 
 func (m *mockedConnections) Stop() {}
+
+func (m *mockedConnections) ExternalAddresses() []string { return nil }
+
+func (m *mockedConnections) AllAddresses() []string { return nil }

lib/api/mocked_model_test.go

@@ -10,7 +10,6 @@ import (
 	"net"
 	"time"
 
-	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/connections"
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/model"
@@ -49,12 +48,12 @@ func (m *mockedModel) ConnectionStats() map[string]interface{} {
 	return nil
 }
 
-func (m *mockedModel) DeviceStatistics() map[string]stats.DeviceStatistics {
-	return nil
+func (m *mockedModel) DeviceStatistics() (map[string]stats.DeviceStatistics, error) {
+	return nil, nil
 }
 
-func (m *mockedModel) FolderStatistics() map[string]stats.FolderStatistics {
-	return nil
+func (m *mockedModel) FolderStatistics() (map[string]stats.FolderStatistics, error) {
+	return nil, nil
 }
 
 func (m *mockedModel) CurrentFolderFile(folder string, file string) (protocol.FileInfo, bool) {
@@ -153,21 +152,29 @@ func (m *mockedModel) LocalChangedFiles(folder string, page, perpage int) []db.F
 	return nil
 }
 
-func (m *mockedModel) Serve()                                                                     {}
-func (m *mockedModel) Stop()                                                                      {}
-func (m *mockedModel) Index(deviceID protocol.DeviceID, folder string, files []protocol.FileInfo) {}
-func (m *mockedModel) IndexUpdate(deviceID protocol.DeviceID, folder string, files []protocol.FileInfo) {
+func (m *mockedModel) Serve() {}
+func (m *mockedModel) Stop()  {}
+
+func (m *mockedModel) Index(deviceID protocol.DeviceID, folder string, files []protocol.FileInfo) error {
+	return nil
+}
+
+func (m *mockedModel) IndexUpdate(deviceID protocol.DeviceID, folder string, files []protocol.FileInfo) error {
+	return nil
 }
 
 func (m *mockedModel) Request(deviceID protocol.DeviceID, folder, name string, size int32, offset int64, hash []byte, weakHash uint32, fromTemporary bool) (protocol.RequestResponse, error) {
 	return nil, nil
 }
 
-func (m *mockedModel) ClusterConfig(deviceID protocol.DeviceID, config protocol.ClusterConfig) {}
+func (m *mockedModel) ClusterConfig(deviceID protocol.DeviceID, config protocol.ClusterConfig) error {
+	return nil
+}
 
 func (m *mockedModel) Closed(conn protocol.Connection, err error) {}
 
-func (m *mockedModel) DownloadProgress(deviceID protocol.DeviceID, folder string, updates []protocol.FileDownloadProgressUpdate) {
+func (m *mockedModel) DownloadProgress(deviceID protocol.DeviceID, folder string, updates []protocol.FileDownloadProgressUpdate) error {
+	return nil
 }
 
 func (m *mockedModel) AddConnection(conn connections.Connection, hello protocol.HelloResult) {}
@@ -180,10 +187,4 @@ func (m *mockedModel) GetHello(protocol.DeviceID) protocol.HelloIntf {
 	return nil
 }
 
-func (m *mockedModel) AddFolder(cfg config.FolderConfiguration) {}
-
-func (m *mockedModel) RestartFolder(from, to config.FolderConfiguration) {}
-
-func (m *mockedModel) StartFolder(folder string) {}
-
 func (m *mockedModel) StartDeadlockDetector(timeout time.Duration) {}

lib/beacon/beacon.go

@@ -7,9 +7,14 @@
 package beacon
 
 import (
+	"context"
+	"fmt"
 	"net"
+	"time"
 
 	"github.com/thejerf/suture"
+
+	"github.com/syncthing/syncthing/lib/util"
 )
 
 type recv struct {
@@ -19,7 +24,93 @@ type recv struct {
 
 type Interface interface {
 	suture.Service
+	fmt.Stringer
 	Send(data []byte)
 	Recv() ([]byte, net.Addr)
 	Error() error
 }
+
+type cast struct {
+	*suture.Supervisor
+	name    string
+	reader  util.ServiceWithError
+	writer  util.ServiceWithError
+	outbox  chan recv
+	inbox   chan []byte
+	stopped chan struct{}
+}
+
+// newCast creates a base object for multi- or broadcasting. Afterwards the
+// caller needs to set reader and writer with the addReader and addWriter
+// methods to get a functional implementation of Interface.
+func newCast(name string) *cast {
+	return &cast{
+		Supervisor: suture.New(name, suture.Spec{
+			// Don't retry too frenetically: an error to open a socket or
+			// whatever is usually something that is either permanent or takes
+			// a while to get solved...
+			FailureThreshold: 2,
+			FailureBackoff:   60 * time.Second,
+			// Only log restarts in debug mode.
+			Log: func(line string) {
+				l.Debugln(line)
+			},
+			PassThroughPanics: true,
+		}),
+		name:    name,
+		inbox:   make(chan []byte),
+		outbox:  make(chan recv, 16),
+		stopped: make(chan struct{}),
+	}
+}
+
+func (c *cast) addReader(svc func(context.Context) error) {
+	c.reader = c.createService(svc, "reader")
+	c.Add(c.reader)
+}
+
+func (c *cast) addWriter(svc func(ctx context.Context) error) {
+	c.writer = c.createService(svc, "writer")
+	c.Add(c.writer)
+}
+
+func (c *cast) createService(svc func(context.Context) error, suffix string) util.ServiceWithError {
+	return util.AsServiceWithError(func(ctx context.Context) error {
+		l.Debugln("Starting", c.name, suffix)
+		err := svc(ctx)
+		l.Debugf("Stopped %v %v: %v", c.name, suffix, err)
+		return err
+	}, fmt.Sprintf("%s/%s", c, suffix))
+}
+
+func (c *cast) Stop() {
+	c.Supervisor.Stop()
+	close(c.stopped)
+}
+
+func (c *cast) String() string {
+	return fmt.Sprintf("%s@%p", c.name, c)
+}
+
+func (c *cast) Send(data []byte) {
+	select {
+	case c.inbox <- data:
+	case <-c.stopped:
+	}
+}
+
+func (c *cast) Recv() ([]byte, net.Addr) {
+	select {
+	case recv := <-c.outbox:
+		return recv.data, recv.src
+	case <-c.stopped:
+	}
+	return nil, nil
+}
+
+func (c *cast) Error() error {
+	if err := c.reader.Error(); err != nil {
+		return err
+	}
+	return c.writer.Error()
+}

lib/beacon/broadcast.go

@@ -7,113 +7,47 @@
 package beacon
 
 import (
-	"fmt"
+	"context"
 	"net"
 	"time"
-
-	"github.com/thejerf/suture"
-
-	"github.com/syncthing/syncthing/lib/util"
 )
 
-type Broadcast struct {
-	*suture.Supervisor
-	port   int
-	inbox  chan []byte
-	outbox chan recv
-	br     *broadcastReader
-	bw     *broadcastWriter
+func NewBroadcast(port int) Interface {
+	c := newCast("broadcastBeacon")
+	c.addReader(func(ctx context.Context) error {
+		return readBroadcasts(ctx, c.outbox, port)
+	})
+	c.addWriter(func(ctx context.Context) error {
+		return writeBroadcasts(ctx, c.inbox, port)
+	})
+	return c
 }
 
-func NewBroadcast(port int) *Broadcast {
-	b := &Broadcast{
-		Supervisor: suture.New("broadcastBeacon", suture.Spec{
-			// Don't retry too frenetically: an error to open a socket or
-			// whatever is usually something that is either permanent or takes
-			// a while to get solved...
-			FailureThreshold: 2,
-			FailureBackoff:   60 * time.Second,
-			// Only log restarts in debug mode.
-			Log: func(line string) {
-				l.Debugln(line)
-			},
-			PassThroughPanics: true,
-		}),
-		port:   port,
-		inbox:  make(chan []byte),
-		outbox: make(chan recv, 16),
-	}
-
-	b.br = &broadcastReader{
-		port:   port,
-		outbox: b.outbox,
-	}
-	b.br.ServiceWithError = util.AsServiceWithError(b.br.serve)
-	b.Add(b.br)
-	b.bw = &broadcastWriter{
-		port:  port,
-		inbox: b.inbox,
-	}
-	b.bw.ServiceWithError = util.AsServiceWithError(b.bw.serve)
-	b.Add(b.bw)
-
-	return b
-}
-
-func (b *Broadcast) Send(data []byte) {
-	b.inbox <- data
-}
-
-func (b *Broadcast) Recv() ([]byte, net.Addr) {
-	recv := <-b.outbox
-	return recv.data, recv.src
-}
-
-func (b *Broadcast) Error() error {
-	if err := b.br.Error(); err != nil {
-		return err
-	}
-	return b.bw.Error()
-}
-
-type broadcastWriter struct {
-	util.ServiceWithError
-	port  int
-	inbox chan []byte
-}
-
-func (w *broadcastWriter) serve(stop chan struct{}) error {
-	l.Debugln(w, "starting")
-	defer l.Debugln(w, "stopping")
-
+func writeBroadcasts(ctx context.Context, inbox <-chan []byte, port int) error {
 	conn, err := net.ListenUDP("udp4", nil)
 	if err != nil {
 		l.Debugln(err)
 		return err
 	}
-	done := make(chan struct{})
-	defer close(done)
+	doneCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	go func() {
-		select {
-		case <-stop:
-		case <-done:
-		}
+		<-doneCtx.Done()
 		conn.Close()
 	}()
 
 	for {
 		var bs []byte
 		select {
-		case bs = <-w.inbox:
-		case <-stop:
+		case bs = <-inbox:
+		case <-doneCtx.Done():
 			return nil
 		}
 
 		addrs, err := net.InterfaceAddrs()
 		if err != nil {
 			l.Debugln(err)
-			w.SetError(err)
-			continue
+			return err
 		}
 
 		var dsts []net.IP
@@ -133,13 +67,13 @@ func (w *broadcastWriter) serve(stop chan struct{}) error {
 
 		success := 0
 		for _, ip := range dsts {
-			dst := &net.UDPAddr{IP: ip, Port: w.port}
+			dst := &net.UDPAddr{IP: ip, Port: port}
 
 			conn.SetWriteDeadline(time.Now().Add(time.Second))
-			_, err := conn.WriteTo(bs, dst)
+			_, err = conn.WriteTo(bs, dst)
 			conn.SetWriteDeadline(time.Time{})
 
-			if err, ok := err.(net.Error); ok && err.Timeout() {
+			if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
 				// Write timeouts should not happen. We treat it as a fatal
 				// error on the socket.
 				l.Debugln(err)
@@ -149,7 +83,6 @@ func (w *broadcastWriter) serve(stop chan struct{}) error {
 			if err != nil {
 				// Some other error that we don't expect. Debug and continue.
 				l.Debugln(err)
-				w.SetError(err)
 				continue
 			}
 
@@ -157,38 +90,24 @@ func (w *broadcastWriter) serve(stop chan struct{}) error {
 			success++
 		}
 
-		if success > 0 {
-			w.SetError(nil)
+		if success == 0 {
+			l.Debugln("couldn't send any braodcasts")
+			return err
 		}
 	}
 }
 
-func (w *broadcastWriter) String() string {
-	return fmt.Sprintf("broadcastWriter@%p", w)
-}
-
-type broadcastReader struct {
-	util.ServiceWithError
-	port   int
-	outbox chan recv
-}
-
-func (r *broadcastReader) serve(stop chan struct{}) error {
-	l.Debugln(r, "starting")
-	defer l.Debugln(r, "stopping")
-
-	conn, err := net.ListenUDP("udp4", &net.UDPAddr{Port: r.port})
+func readBroadcasts(ctx context.Context, outbox chan<- recv, port int) error {
+	conn, err := net.ListenUDP("udp4", &net.UDPAddr{Port: port})
 	if err != nil {
 		l.Debugln(err)
 		return err
 	}
-	done := make(chan struct{})
-	defer close(done)
+
+	doneCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	go func() {
-		select {
-		case <-stop:
-		case <-done:
-		}
+		<-doneCtx.Done()
 		conn.Close()
 	}()
 
@@ -200,15 +119,13 @@ func (r *broadcastReader) serve(stop chan struct{}) error {
 			return err
 		}
 
-		r.SetError(nil)
-
 		l.Debugf("recv %d bytes from %s", n, addr)
 
 		c := make([]byte, n)
 		copy(c, bs)
 		select {
-		case r.outbox <- recv{c, addr}:
-		case <-stop:
+		case outbox <- recv{c, addr}:
+		case <-doneCtx.Done():
 			return nil
 		default:
 			l.Debugln("dropping message")
@@ -216,10 +133,6 @@ func (r *broadcastReader) serve(stop chan struct{}) error {
 	}
 }
 
-func (r *broadcastReader) String() string {
-	return fmt.Sprintf("broadcastReader@%p", r)
-}
-
 func bcast(ip *net.IPNet) *net.IPNet {
 	var bc = &net.IPNet{}
 	bc.IP = make([]byte, len(ip.IP))

lib/beacon/debug.go

@@ -7,16 +7,9 @@
 package beacon
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
 	l = logger.DefaultLogger.NewFacility("beacon", "Multicast and broadcast discovery")
 )
-
-func init() {
-	l.SetDebug("beacon", strings.Contains(os.Getenv("STTRACE"), "beacon") || os.Getenv("STTRACE") == "all")
-}

lib/beacon/multicast.go

@@ -7,87 +7,27 @@
 package beacon
 
 import (
+	"context"
 	"errors"
-	"fmt"
 	"net"
 	"time"
 
-	"github.com/thejerf/suture"
 	"golang.org/x/net/ipv6"
-
-	"github.com/syncthing/syncthing/lib/util"
 )
 
-type Multicast struct {
-	*suture.Supervisor
-	inbox  chan []byte
-	outbox chan recv
-	mr     *multicastReader
-	mw     *multicastWriter
+func NewMulticast(addr string) Interface {
+	c := newCast("multicastBeacon")
+	c.addReader(func(ctx context.Context) error {
+		return readMulticasts(ctx, c.outbox, addr)
+	})
+	c.addWriter(func(ctx context.Context) error {
+		return writeMulticasts(ctx, c.inbox, addr)
+	})
+	return c
 }
 
-func NewMulticast(addr string) *Multicast {
-	m := &Multicast{
-		Supervisor: suture.New("multicastBeacon", suture.Spec{
-			// Don't retry too frenetically: an error to open a socket or
-			// whatever is usually something that is either permanent or takes
-			// a while to get solved...
-			FailureThreshold: 2,
-			FailureBackoff:   60 * time.Second,
-			// Only log restarts in debug mode.
-			Log: func(line string) {
-				l.Debugln(line)
-			},
-			PassThroughPanics: true,
-		}),
-		inbox:  make(chan []byte),
-		outbox: make(chan recv, 16),
-	}
-
-	m.mr = &multicastReader{
-		addr:   addr,
-		outbox: m.outbox,
-	}
-	m.mr.ServiceWithError = util.AsServiceWithError(m.mr.serve)
-	m.Add(m.mr)
-
-	m.mw = &multicastWriter{
-		addr:  addr,
-		inbox: m.inbox,
-	}
-	m.mw.ServiceWithError = util.AsServiceWithError(m.mw.serve)
-	m.Add(m.mw)
-
-	return m
-}
-
-func (m *Multicast) Send(data []byte) {
-	m.inbox <- data
-}
-
-func (m *Multicast) Recv() ([]byte, net.Addr) {
-	recv := <-m.outbox
-	return recv.data, recv.src
-}
-
-func (m *Multicast) Error() error {
-	if err := m.mr.Error(); err != nil {
-		return err
-	}
-	return m.mw.Error()
-}
-
-type multicastWriter struct {
-	util.ServiceWithError
-	addr  string
-	inbox <-chan []byte
-}
-
-func (w *multicastWriter) serve(stop chan struct{}) error {
-	l.Debugln(w, "starting")
-	defer l.Debugln(w, "stopping")
-
-	gaddr, err := net.ResolveUDPAddr("udp6", w.addr)
+func writeMulticasts(ctx context.Context, inbox <-chan []byte, addr string) error {
+	gaddr, err := net.ResolveUDPAddr("udp6", addr)
 	if err != nil {
 		l.Debugln(err)
 		return err
@@ -98,13 +38,10 @@ func (w *multicastWriter) serve(stop chan struct{}) error {
 		l.Debugln(err)
 		return err
 	}
-	done := make(chan struct{})
-	defer close(done)
+	doneCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	go func() {
-		select {
-		case <-stop:
-		case <-done:
-		}
+		<-doneCtx.Done()
 		conn.Close()
 	}()
 
@@ -117,8 +54,8 @@ func (w *multicastWriter) serve(stop chan struct{}) error {
 	for {
 		var bs []byte
 		select {
-		case bs = <-w.inbox:
-		case <-stop:
+		case bs = <-inbox:
+		case <-doneCtx.Done():
 			return nil
 		}
 
@@ -137,7 +74,6 @@ func (w *multicastWriter) serve(stop chan struct{}) error {
 
 			if err != nil {
 				l.Debugln(err, "on write to", gaddr, intf.Name)
-				w.SetError(err)
 				continue
 			}
 
@@ -146,50 +82,34 @@ func (w *multicastWriter) serve(stop chan struct{}) error {
 			success++
 
 			select {
-			case <-stop:
+			case <-doneCtx.Done():
 				return nil
 			default:
 			}
 		}
 
-		if success > 0 {
-			w.SetError(nil)
+		if success == 0 {
+			return err
 		}
 	}
 }
 
-func (w *multicastWriter) String() string {
-	return fmt.Sprintf("multicastWriter@%p", w)
-}
-
-type multicastReader struct {
-	util.ServiceWithError
-	addr   string
-	outbox chan<- recv
-}
-
-func (r *multicastReader) serve(stop chan struct{}) error {
-	l.Debugln(r, "starting")
-	defer l.Debugln(r, "stopping")
-
-	gaddr, err := net.ResolveUDPAddr("udp6", r.addr)
+func readMulticasts(ctx context.Context, outbox chan<- recv, addr string) error {
+	gaddr, err := net.ResolveUDPAddr("udp6", addr)
 	if err != nil {
 		l.Debugln(err)
 		return err
 	}
 
-	conn, err := net.ListenPacket("udp6", r.addr)
+	conn, err := net.ListenPacket("udp6", addr)
 	if err != nil {
 		l.Debugln(err)
 		return err
 	}
-	done := make(chan struct{})
-	defer close(done)
+	doneCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	go func() {
-		select {
-		case <-stop:
-		case <-done:
-		}
+		<-doneCtx.Done()
 		conn.Close()
 	}()
 
@@ -219,28 +139,23 @@ func (r *multicastReader) serve(stop chan struct{}) error {
 	bs := make([]byte, 65536)
 	for {
 		select {
-		case <-stop:
+		case <-doneCtx.Done():
 			return nil
 		default:
 		}
 		n, _, addr, err := pconn.ReadFrom(bs)
 		if err != nil {
 			l.Debugln(err)
-			r.SetError(err)
-			continue
+			return err
 		}
 		l.Debugf("recv %d bytes from %s", n, addr)
 
 		c := make([]byte, n)
 		copy(c, bs)
 		select {
-		case r.outbox <- recv{c, addr}:
+		case outbox <- recv{c, addr}:
 		default:
 			l.Debugln("dropping message")
 		}
 	}
 }
-
-func (r *multicastReader) String() string {
-	return fmt.Sprintf("multicastReader@%p", r)
-}

lib/build/build.go

@@ -18,10 +18,11 @@ import (
 
 var (
 	// Injected by build script
+	Program = "syncthing"
 	Version = "unknown-dev"
-	Host    = "unknown" // Set by build script
-	User    = "unknown" // Set by build script
-	Stamp   = "0"       // Set by build script
+	Host    = "unknown"
+	User    = "unknown"
+	Stamp   = "0"
 
 	// Static
 	Codename = "Fermium Flea"
@@ -73,7 +74,7 @@ func setBuildData() {
 	Date = time.Unix(int64(stamp), 0)
 
 	date := Date.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`syncthing %s "%s" (%s %s-%s) %s@%s %s`, Version, Codename, runtime.Version(), runtime.GOOS, runtime.GOARCH, User, Host, date)
+	LongVersion = fmt.Sprintf(`%s %s "%s" (%s %s-%s) %s@%s %s`, Program, Version, Codename, runtime.Version(), runtime.GOOS, runtime.GOARCH, User, Host, date)
 
 	if len(Tags) > 0 {
 		LongVersion = fmt.Sprintf("%s [%s]", LongVersion, strings.Join(Tags, ", "))

lib/config/commit_test.go

@@ -44,7 +44,7 @@ func (validationError) String() string {
 func TestReplaceCommit(t *testing.T) {
 	t.Skip("broken, fails randomly, #3834")
 
-	w := Wrap("/dev/null", Configuration{Version: 0})
+	w := wrap("/dev/null", Configuration{Version: 0})
 	if w.RawCopy().Version != 0 {
 		t.Fatal("Config incorrect")
 	}

lib/config/config.go

@@ -10,7 +10,6 @@ package config
 import (
 	"encoding/json"
 	"encoding/xml"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -22,6 +21,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/rand"
@@ -87,7 +88,6 @@ var (
 		"stun.voiparound.com:3478",
 		"stun.voipbuster.com:3478",
 		"stun.voipstunt.com:3478",
-		"stun.voxgratia.org:3478",
 		"stun.xten.com:3478",
 	}
 )
@@ -109,7 +109,8 @@ func New(myID protocol.DeviceID) Configuration {
 
 	// Can't happen.
 	if err := cfg.prepare(myID); err != nil {
-		panic("bug: error in preparing new folder: " + err.Error())
+		l.Warnln("bug: error in preparing new folder:", err)
+		panic("error in preparing new folder")
 	}
 
 	return cfg
@@ -120,18 +121,18 @@ func NewWithFreePorts(myID protocol.DeviceID) (Configuration, error) {
 
 	port, err := getFreePort("127.0.0.1", DefaultGUIPort)
 	if err != nil {
-		return Configuration{}, fmt.Errorf("get free port (GUI): %v", err)
+		return Configuration{}, errors.Wrap(err, "get free port (GUI)")
 	}
 	cfg.GUI.RawAddress = fmt.Sprintf("127.0.0.1:%d", port)
 
 	port, err = getFreePort("0.0.0.0", DefaultTCPPort)
 	if err != nil {
-		return Configuration{}, fmt.Errorf("get free port (BEP): %v", err)
+		return Configuration{}, errors.Wrap(err, "get free port (BEP)")
 	}
 	if port == DefaultTCPPort {
-		cfg.Options.ListenAddresses = []string{"default"}
+		cfg.Options.RawListenAddresses = []string{"default"}
 	} else {
-		cfg.Options.ListenAddresses = []string{
+		cfg.Options.RawListenAddresses = []string{
 			fmt.Sprintf("tcp://%s", net.JoinHostPort("0.0.0.0", strconv.Itoa(port))),
 			"dynamic+https://relays.syncthing.net/endpoint",
 		}
@@ -304,8 +305,8 @@ func (cfg *Configuration) clean() error {
 		existingFolders[folder.ID] = folder
 	}
 
-	cfg.Options.ListenAddresses = util.UniqueTrimmedStrings(cfg.Options.ListenAddresses)
-	cfg.Options.GlobalAnnServers = util.UniqueTrimmedStrings(cfg.Options.GlobalAnnServers)
+	cfg.Options.RawListenAddresses = util.UniqueTrimmedStrings(cfg.Options.RawListenAddresses)
+	cfg.Options.RawGlobalAnnServers = util.UniqueTrimmedStrings(cfg.Options.RawGlobalAnnServers)
 
 	if cfg.Version > 0 && cfg.Version < OldestHandledVersion {
 		l.Warnf("Configuration version %d is deprecated. Attempting best effort conversion, but please verify manually.", cfg.Version)
@@ -395,7 +396,7 @@ nextPendingDevice:
 	// Deprecated protocols are removed from the list of listeners and
 	// device addresses. So far just kcp*.
 	for _, prefix := range []string{"kcp"} {
-		cfg.Options.ListenAddresses = filterURLSchemePrefix(cfg.Options.ListenAddresses, prefix)
+		cfg.Options.RawListenAddresses = filterURLSchemePrefix(cfg.Options.RawListenAddresses, prefix)
 		for i := range cfg.Devices {
 			dev := &cfg.Devices[i]
 			dev.Addresses = filterURLSchemePrefix(dev.Addresses, prefix)

lib/config/config_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/d4l3k/messagediff"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
@@ -36,8 +37,8 @@ func init() {
 
 func TestDefaultValues(t *testing.T) {
 	expected := OptionsConfiguration{
-		ListenAddresses:         []string{"default"},
-		GlobalAnnServers:        []string{"default"},
+		RawListenAddresses:      []string{"default"},
+		RawGlobalAnnServers:     []string{"default"},
 		GlobalAnnEnabled:        true,
 		LocalAnnEnabled:         true,
 		LocalAnnPort:            21027,
@@ -73,7 +74,7 @@ func TestDefaultValues(t *testing.T) {
 		CREnabled:               true,
 		StunKeepaliveStartS:     180,
 		StunKeepaliveMinS:       20,
-		StunServers:             []string{"default"},
+		RawStunServers:          []string{"default"},
 	}
 
 	cfg := New(device1)
@@ -86,7 +87,7 @@ func TestDefaultValues(t *testing.T) {
 func TestDeviceConfig(t *testing.T) {
 	for i := OldestHandledVersion; i <= CurrentVersion; i++ {
 		os.RemoveAll(filepath.Join("testdata", DefaultMarkerName))
-		wr, err := Load(fmt.Sprintf("testdata/v%d.xml", i), device1)
+		wr, err := load(fmt.Sprintf("testdata/v%d.xml", i), device1)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -168,22 +169,22 @@ func TestDeviceConfig(t *testing.T) {
 }
 
 func TestNoListenAddresses(t *testing.T) {
-	cfg, err := Load("testdata/nolistenaddress.xml", device1)
+	cfg, err := load("testdata/nolistenaddress.xml", device1)
 	if err != nil {
 		t.Error(err)
 	}
 
 	expected := []string{""}
-	actual := cfg.Options().ListenAddresses
+	actual := cfg.Options().RawListenAddresses
 	if diff, equal := messagediff.PrettyDiff(expected, actual); !equal {
-		t.Errorf("Unexpected ListenAddresses. Diff:\n%s", diff)
+		t.Errorf("Unexpected RawListenAddresses. Diff:\n%s", diff)
 	}
 }
 
 func TestOverriddenValues(t *testing.T) {
 	expected := OptionsConfiguration{
-		ListenAddresses:         []string{"tcp://:23000"},
-		GlobalAnnServers:        []string{"udp4://syncthing.nym.se:22026"},
+		RawListenAddresses:      []string{"tcp://:23000"},
+		RawGlobalAnnServers:     []string{"udp4://syncthing.nym.se:22026"},
 		GlobalAnnEnabled:        false,
 		LocalAnnEnabled:         false,
 		LocalAnnPort:            42123,
@@ -221,11 +222,11 @@ func TestOverriddenValues(t *testing.T) {
 		CREnabled:               false,
 		StunKeepaliveStartS:     9000,
 		StunKeepaliveMinS:       900,
-		StunServers:             []string{"foo"},
+		RawStunServers:          []string{"foo"},
 	}
 
 	os.Unsetenv("STNOUPGRADE")
-	cfg, err := Load("testdata/overridenvalues.xml", device1)
+	cfg, err := load("testdata/overridenvalues.xml", device1)
 	if err != nil {
 		t.Error(err)
 	}
@@ -270,7 +271,7 @@ func TestDeviceAddressesDynamic(t *testing.T) {
 		},
 	}
 
-	cfg, err := Load("testdata/deviceaddressesdynamic.xml", device4)
+	cfg, err := load("testdata/deviceaddressesdynamic.xml", device4)
 	if err != nil {
 		t.Error(err)
 	}
@@ -319,7 +320,7 @@ func TestDeviceCompression(t *testing.T) {
 		},
 	}
 
-	cfg, err := Load("testdata/devicecompression.xml", device4)
+	cfg, err := load("testdata/devicecompression.xml", device4)
 	if err != nil {
 		t.Error(err)
 	}
@@ -365,7 +366,7 @@ func TestDeviceAddressesStatic(t *testing.T) {
 		},
 	}
 
-	cfg, err := Load("testdata/deviceaddressesstatic.xml", device4)
+	cfg, err := load("testdata/deviceaddressesstatic.xml", device4)
 	if err != nil {
 		t.Error(err)
 	}
@@ -377,7 +378,7 @@ func TestDeviceAddressesStatic(t *testing.T) {
 }
 
 func TestVersioningConfig(t *testing.T) {
-	cfg, err := Load("testdata/versioningconfig.xml", device4)
+	cfg, err := load("testdata/versioningconfig.xml", device4)
 	if err != nil {
 		t.Error(err)
 	}
@@ -404,7 +405,7 @@ func TestIssue1262(t *testing.T) {
 		t.Skipf("path gets converted to absolute as part of the filesystem initialization on linux")
 	}
 
-	cfg, err := Load("testdata/issue-1262.xml", device4)
+	cfg, err := load("testdata/issue-1262.xml", device4)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -418,25 +419,25 @@ func TestIssue1262(t *testing.T) {
 }
 
 func TestIssue1750(t *testing.T) {
-	cfg, err := Load("testdata/issue-1750.xml", device4)
+	cfg, err := load("testdata/issue-1750.xml", device4)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if cfg.Options().ListenAddresses[0] != "tcp://:23000" {
-		t.Errorf("%q != %q", cfg.Options().ListenAddresses[0], "tcp://:23000")
+	if cfg.Options().RawListenAddresses[0] != "tcp://:23000" {
+		t.Errorf("%q != %q", cfg.Options().RawListenAddresses[0], "tcp://:23000")
 	}
 
-	if cfg.Options().ListenAddresses[1] != "tcp://:23001" {
-		t.Errorf("%q != %q", cfg.Options().ListenAddresses[1], "tcp://:23001")
+	if cfg.Options().RawListenAddresses[1] != "tcp://:23001" {
+		t.Errorf("%q != %q", cfg.Options().RawListenAddresses[1], "tcp://:23001")
 	}
 
-	if cfg.Options().GlobalAnnServers[0] != "udp4://syncthing.nym.se:22026" {
-		t.Errorf("%q != %q", cfg.Options().GlobalAnnServers[0], "udp4://syncthing.nym.se:22026")
+	if cfg.Options().RawGlobalAnnServers[0] != "udp4://syncthing.nym.se:22026" {
+		t.Errorf("%q != %q", cfg.Options().RawGlobalAnnServers[0], "udp4://syncthing.nym.se:22026")
 	}
 
-	if cfg.Options().GlobalAnnServers[1] != "udp4://syncthing.nym.se:22027" {
-		t.Errorf("%q != %q", cfg.Options().GlobalAnnServers[1], "udp4://syncthing.nym.se:22027")
+	if cfg.Options().RawGlobalAnnServers[1] != "udp4://syncthing.nym.se:22027" {
+		t.Errorf("%q != %q", cfg.Options().RawGlobalAnnServers[1], "udp4://syncthing.nym.se:22027")
 	}
 }
 
@@ -520,7 +521,7 @@ func TestNewSaveLoad(t *testing.T) {
 	}
 
 	intCfg := New(device1)
-	cfg := Wrap(path, intCfg)
+	cfg := wrap(path, intCfg)
 
 	// To make the equality pass later
 	cfg.(*wrapper).cfg.XMLName.Local = "configuration"
@@ -537,7 +538,7 @@ func TestNewSaveLoad(t *testing.T) {
 		t.Error(path, "does not exist")
 	}
 
-	cfg2, err := Load(path, device1)
+	cfg2, err := load(path, device1)
 	if err != nil {
 		t.Error(err)
 	}
@@ -552,19 +553,19 @@ func TestNewSaveLoad(t *testing.T) {
 func TestPrepare(t *testing.T) {
 	var cfg Configuration
 
-	if cfg.Folders != nil || cfg.Devices != nil || cfg.Options.ListenAddresses != nil {
+	if cfg.Folders != nil || cfg.Devices != nil || cfg.Options.RawListenAddresses != nil {
 		t.Error("Expected nil")
 	}
 
 	cfg.prepare(device1)
 
-	if cfg.Folders == nil || cfg.Devices == nil || cfg.Options.ListenAddresses == nil {
+	if cfg.Folders == nil || cfg.Devices == nil || cfg.Options.RawListenAddresses == nil {
 		t.Error("Unexpected nil")
 	}
 }
 
 func TestCopy(t *testing.T) {
-	wrapper, err := Load("testdata/example.xml", device1)
+	wrapper, err := load("testdata/example.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -579,7 +580,7 @@ func TestCopy(t *testing.T) {
 
 	cfg.Devices[0].Addresses[0] = "wrong"
 	cfg.Folders[0].Devices[0].DeviceID = protocol.DeviceID{0, 1, 2, 3}
-	cfg.Options.ListenAddresses[0] = "wrong"
+	cfg.Options.RawListenAddresses[0] = "wrong"
 	cfg.GUI.APIKey = "wrong"
 
 	bsChanged, err := json.MarshalIndent(cfg, "", "  ")
@@ -603,7 +604,7 @@ func TestCopy(t *testing.T) {
 }
 
 func TestPullOrder(t *testing.T) {
-	wrapper, err := Load("testdata/pullorder.xml", device1)
+	wrapper, err := load("testdata/pullorder.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -643,7 +644,7 @@ func TestPullOrder(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	wrapper = Wrap("testdata/pullorder.xml", cfg)
+	wrapper = wrap("testdata/pullorder.xml", cfg)
 	folders = wrapper.Folders()
 
 	for _, tc := range expected {
@@ -654,7 +655,7 @@ func TestPullOrder(t *testing.T) {
 }
 
 func TestLargeRescanInterval(t *testing.T) {
-	wrapper, err := Load("testdata/largeinterval.xml", device1)
+	wrapper, err := load("testdata/largeinterval.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -692,7 +693,7 @@ func TestGUIConfigURL(t *testing.T) {
 func TestDuplicateDevices(t *testing.T) {
 	// Duplicate devices should be removed
 
-	wrapper, err := Load("testdata/dupdevices.xml", device1)
+	wrapper, err := load("testdata/dupdevices.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -710,7 +711,7 @@ func TestDuplicateDevices(t *testing.T) {
 func TestDuplicateFolders(t *testing.T) {
 	// Duplicate folders are a loading error
 
-	_, err := Load("testdata/dupfolders.xml", device1)
+	_, err := load("testdata/dupfolders.xml", device1)
 	if err == nil || !strings.Contains(err.Error(), errFolderIDDuplicate.Error()) {
 		t.Fatal(`Expected error to mention "duplicate folder ID":`, err)
 	}
@@ -721,7 +722,7 @@ func TestEmptyFolderPaths(t *testing.T) {
 	// get messed up by the prepare steps (e.g., become the current dir or
 	// get a slash added so that it becomes the root directory or similar).
 
-	_, err := Load("testdata/nopath.xml", device1)
+	_, err := load("testdata/nopath.xml", device1)
 	if err == nil || !strings.Contains(err.Error(), errFolderPathEmpty.Error()) {
 		t.Fatal("Expected error due to empty folder path, got", err)
 	}
@@ -770,7 +771,7 @@ func TestV14ListenAddressesMigration(t *testing.T) {
 		cfg := Configuration{
 			Version: 13,
 			Options: OptionsConfiguration{
-				ListenAddresses:        tc[0],
+				RawListenAddresses:     tc[0],
 				DeprecatedRelayServers: tc[1],
 			},
 		}
@@ -780,8 +781,8 @@ func TestV14ListenAddressesMigration(t *testing.T) {
 		}
 
 		sort.Strings(tc[2])
-		if !reflect.DeepEqual(cfg.Options.ListenAddresses, tc[2]) {
-			t.Errorf("Migration error; actual %#v != expected %#v", cfg.Options.ListenAddresses, tc[2])
+		if !reflect.DeepEqual(cfg.Options.RawListenAddresses, tc[2]) {
+			t.Errorf("Migration error; actual %#v != expected %#v", cfg.Options.RawListenAddresses, tc[2])
 		}
 	}
 }
@@ -790,7 +791,7 @@ func TestIgnoredDevices(t *testing.T) {
 	// Verify that ignored devices that are also present in the
 	// configuration are not in fact ignored.
 
-	wrapper, err := Load("testdata/ignoreddevices.xml", device1)
+	wrapper, err := load("testdata/ignoreddevices.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -808,7 +809,7 @@ func TestIgnoredFolders(t *testing.T) {
 	// configuration are not in fact ignored.
 	// Also, verify that folders that are shared with a device are not ignored.
 
-	wrapper, err := Load("testdata/ignoredfolders.xml", device1)
+	wrapper, err := load("testdata/ignoredfolders.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -844,7 +845,7 @@ func TestIgnoredFolders(t *testing.T) {
 func TestGetDevice(t *testing.T) {
 	// Verify that the Device() call does the right thing
 
-	wrapper, err := Load("testdata/ignoreddevices.xml", device1)
+	wrapper, err := load("testdata/ignoreddevices.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -871,7 +872,7 @@ func TestGetDevice(t *testing.T) {
 }
 
 func TestSharesRemovedOnDeviceRemoval(t *testing.T) {
-	wrapper, err := Load("testdata/example.xml", device1)
+	wrapper, err := load("testdata/example.xml", device1)
 	if err != nil {
 		t.Errorf("Failed: %s", err)
 	}
@@ -956,7 +957,7 @@ func TestIssue4219(t *testing.T) {
 		t.Errorf("There should be three ignored folders, not %d", ignoredFolders)
 	}
 
-	w := Wrap("/tmp/cfg", cfg)
+	w := wrap("/tmp/cfg", cfg)
 	if !w.IgnoredFolder(device2, "t1") {
 		t.Error("Folder device2 t1 should be ignored")
 	}
@@ -1145,3 +1146,11 @@ func defaultConfigAsMap() map[string]interface{} {
 	}
 	return tmp
 }
+
+func load(path string, myID protocol.DeviceID) (Wrapper, error) {
+	return Load(path, myID, events.NoopLogger)
+}
+
+func wrap(path string, cfg Configuration) Wrapper {
+	return Wrap(path, cfg, events.NoopLogger)
+}

lib/config/debug.go

@@ -7,16 +7,9 @@
 package config
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
 	l = logger.DefaultLogger.NewFacility("config", "Configuration loading and saving")
 )
-
-func init() {
-	l.SetDebug("config", strings.Contains(os.Getenv("STTRACE"), "config") || os.Getenv("STTRACE") == "all")
-}

lib/config/folderconfiguration.go

@@ -10,11 +10,14 @@ import (
 	"errors"
 	"fmt"
 	"runtime"
+	"strings"
+	"time"
+
+	"github.com/shirou/gopsutil/disk"
 
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/util"
-	"github.com/syncthing/syncthing/lib/versioner"
 )
 
 var (
@@ -53,8 +56,10 @@ type FolderConfiguration struct {
 	WeakHashThresholdPct    int                         `xml:"weakHashThresholdPct" json:"weakHashThresholdPct"` // Use weak hash if more than X percent of the file has changed. Set to -1 to always use weak hash.
 	MarkerName              string                      `xml:"markerName" json:"markerName"`
 	CopyOwnershipFromParent bool                        `xml:"copyOwnershipFromParent" json:"copyOwnershipFromParent"`
+	RawModTimeWindowS       int                         `xml:"modTimeWindowS" json:"modTimeWindowS"`
 
-	cachedFilesystem fs.Filesystem
+	cachedFilesystem    fs.Filesystem
+	cachedModTimeWindow time.Duration
 
 	DeprecatedReadOnly       bool    `xml:"ro,attr,omitempty" json:"-"`
 	DeprecatedMinDiskFreePct float64 `xml:"minDiskFreePct,omitempty" json:"-"`
@@ -99,16 +104,8 @@ func (f FolderConfiguration) Filesystem() fs.Filesystem {
 	return f.cachedFilesystem
 }
 
-func (f FolderConfiguration) Versioner() versioner.Versioner {
-	if f.Versioning.Type == "" {
-		return nil
-	}
-	versionerFactory, ok := versioner.Factories[f.Versioning.Type]
-	if !ok {
-		panic(fmt.Sprintf("Requested versioning type %q that does not exist", f.Versioning.Type))
-	}
-
-	return versionerFactory(f.ID, f.Filesystem(), f.Versioning.Params)
+func (f FolderConfiguration) ModTimeWindow() time.Duration {
+	return f.cachedModTimeWindow
 }
 
 func (f *FolderConfiguration) CreateMarker() error {
@@ -233,6 +230,21 @@ func (f *FolderConfiguration) prepare() {
 	if f.MarkerName == "" {
 		f.MarkerName = DefaultMarkerName
 	}
+
+	switch {
+	case f.RawModTimeWindowS > 0:
+		f.cachedModTimeWindow = time.Duration(f.RawModTimeWindowS) * time.Second
+	case runtime.GOOS == "android":
+		if usage, err := disk.Usage(f.Filesystem().URI()); err != nil {
+			f.cachedModTimeWindow = 2 * time.Second
+			l.Debugf(`Detecting FS at "%v" on android: Setting mtime window to 2s: err == "%v"`, f.Path, err)
+		} else if usage.Fstype == "" || strings.Contains(strings.ToLower(usage.Fstype), "fat") {
+			f.cachedModTimeWindow = 2 * time.Second
+			l.Debugf(`Detecting FS at "%v" on android: Setting mtime window to 2s: usage.Fstype == "%v"`, f.Path, usage.Fstype)
+		} else {
+			l.Debugf(`Detecting FS at %v on android: Leaving mtime window at 0: usage.Fstype == "%v"`, f.Path, usage.Fstype)
+		}
+	}
 }
 
 // RequiresRestartOnly returns a copy with only the attributes that require

lib/config/migrations.go

@@ -216,12 +216,12 @@ func migrateToConfigV18(cfg *Configuration) {
 func migrateToConfigV15(cfg *Configuration) {
 	// Undo v0.13.0 broken migration
 
-	for i, addr := range cfg.Options.GlobalAnnServers {
+	for i, addr := range cfg.Options.RawGlobalAnnServers {
 		switch addr {
 		case "default-v4v2/":
-			cfg.Options.GlobalAnnServers[i] = "default-v4"
+			cfg.Options.RawGlobalAnnServers[i] = "default-v4"
 		case "default-v6v2/":
-			cfg.Options.GlobalAnnServers[i] = "default-v6"
+			cfg.Options.RawGlobalAnnServers[i] = "default-v6"
 		}
 	}
 }
@@ -248,9 +248,9 @@ func migrateToConfigV14(cfg *Configuration) {
 	hasDefault := false
 	for _, raddr := range cfg.Options.DeprecatedRelayServers {
 		if raddr == "dynamic+https://relays.syncthing.net/endpoint" {
-			for i, addr := range cfg.Options.ListenAddresses {
+			for i, addr := range cfg.Options.RawListenAddresses {
 				if addr == "tcp://0.0.0.0:22000" {
-					cfg.Options.ListenAddresses[i] = "default"
+					cfg.Options.RawListenAddresses[i] = "default"
 					hasDefault = true
 					break
 				}
@@ -269,16 +269,16 @@ func migrateToConfigV14(cfg *Configuration) {
 		if addr == "" {
 			continue
 		}
-		cfg.Options.ListenAddresses = append(cfg.Options.ListenAddresses, addr)
+		cfg.Options.RawListenAddresses = append(cfg.Options.RawListenAddresses, addr)
 	}
 
 	cfg.Options.DeprecatedRelayServers = nil
 
 	// For consistency
-	sort.Strings(cfg.Options.ListenAddresses)
+	sort.Strings(cfg.Options.RawListenAddresses)
 
 	var newAddrs []string
-	for _, addr := range cfg.Options.GlobalAnnServers {
+	for _, addr := range cfg.Options.RawGlobalAnnServers {
 		uri, err := url.Parse(addr)
 		if err != nil {
 			// That's odd. Skip the broken address.
@@ -291,7 +291,7 @@ func migrateToConfigV14(cfg *Configuration) {
 
 		newAddrs = append(newAddrs, addr)
 	}
-	cfg.Options.GlobalAnnServers = newAddrs
+	cfg.Options.RawGlobalAnnServers = newAddrs
 
 	for i, fcfg := range cfg.Folders {
 		if fcfg.DeprecatedReadOnly {
@@ -315,9 +315,9 @@ func migrateToConfigV13(cfg *Configuration) {
 
 func migrateToConfigV12(cfg *Configuration) {
 	// Change listen address schema
-	for i, addr := range cfg.Options.ListenAddresses {
+	for i, addr := range cfg.Options.RawListenAddresses {
 		if len(addr) > 0 && !strings.HasPrefix(addr, "tcp://") {
-			cfg.Options.ListenAddresses[i] = util.Address("tcp", addr)
+			cfg.Options.RawListenAddresses[i] = util.Address("tcp", addr)
 		}
 	}
 
@@ -332,7 +332,7 @@ func migrateToConfigV12(cfg *Configuration) {
 	// Use new discovery server
 	var newDiscoServers []string
 	var useDefault bool
-	for _, addr := range cfg.Options.GlobalAnnServers {
+	for _, addr := range cfg.Options.RawGlobalAnnServers {
 		if addr == "udp4://announce.syncthing.net:22026" {
 			useDefault = true
 		} else if addr == "udp6://announce-v6.syncthing.net:22026" {
@@ -344,7 +344,7 @@ func migrateToConfigV12(cfg *Configuration) {
 	if useDefault {
 		newDiscoServers = append(newDiscoServers, "default")
 	}
-	cfg.Options.GlobalAnnServers = newDiscoServers
+	cfg.Options.RawGlobalAnnServers = newDiscoServers
 
 	// Use new multicast group
 	if cfg.Options.LocalAnnMCAddr == "[ff32::5222]:21026" {

lib/config/optionsconfiguration.go

@@ -9,12 +9,13 @@ package config
 import (
 	"fmt"
 
+	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/util"
 )
 
 type OptionsConfiguration struct {
-	ListenAddresses         []string `xml:"listenAddress" json:"listenAddresses" default:"default"`
-	GlobalAnnServers        []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" default:"default" restart:"true"`
+	RawListenAddresses      []string `xml:"listenAddress" json:"listenAddresses" default:"default"`
+	RawGlobalAnnServers     []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" default:"default" restart:"true"`
 	GlobalAnnEnabled        bool     `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true" restart:"true"`
 	LocalAnnEnabled         bool     `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true" restart:"true"`
 	LocalAnnPort            int      `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027" restart:"true"`
@@ -56,7 +57,8 @@ type OptionsConfiguration struct {
 	CREnabled               bool     `xml:"crashReportingEnabled" json:"crashReportingEnabled" default:"true" restart:"true"`
 	StunKeepaliveStartS     int      `xml:"stunKeepaliveStartS" json:"stunKeepaliveStartS" default:"180"` // 0 for off
 	StunKeepaliveMinS       int      `xml:"stunKeepaliveMinS" json:"stunKeepaliveMinS" default:"20"`      // 0 for off
-	StunServers             []string `xml:"stunServer" json:"stunServers" default:"default"`
+	RawStunServers          []string `xml:"stunServer" json:"stunServers" default:"default"`
+	DatabaseTuning          Tuning   `xml:"databaseTuning" json:"databaseTuning" restart:"true"`
 
 	DeprecatedUPnPEnabled        bool     `xml:"upnpEnabled,omitempty" json:"-"`
 	DeprecatedUPnPLeaseM         int      `xml:"upnpLeaseMinutes,omitempty" json:"-"`
@@ -68,10 +70,10 @@ type OptionsConfiguration struct {
 
 func (opts OptionsConfiguration) Copy() OptionsConfiguration {
 	optsCopy := opts
-	optsCopy.ListenAddresses = make([]string, len(opts.ListenAddresses))
-	copy(optsCopy.ListenAddresses, opts.ListenAddresses)
-	optsCopy.GlobalAnnServers = make([]string, len(opts.GlobalAnnServers))
-	copy(optsCopy.GlobalAnnServers, opts.GlobalAnnServers)
+	optsCopy.RawListenAddresses = make([]string, len(opts.RawListenAddresses))
+	copy(optsCopy.RawListenAddresses, opts.RawListenAddresses)
+	optsCopy.RawGlobalAnnServers = make([]string, len(opts.RawGlobalAnnServers))
+	copy(optsCopy.RawGlobalAnnServers, opts.RawGlobalAnnServers)
 	optsCopy.AlwaysLocalNets = make([]string, len(opts.AlwaysLocalNets))
 	copy(optsCopy.AlwaysLocalNets, opts.AlwaysLocalNets)
 	optsCopy.UnackedNotificationIDs = make([]string, len(opts.UnackedNotificationIDs))
@@ -96,3 +98,57 @@ func (opts OptionsConfiguration) RequiresRestartOnly() OptionsConfiguration {
 func (opts OptionsConfiguration) IsStunDisabled() bool {
 	return opts.StunKeepaliveMinS < 1 || opts.StunKeepaliveStartS < 1 || !opts.NATEnabled
 }
+
+func (opts OptionsConfiguration) ListenAddresses() []string {
+	var addresses []string
+	for _, addr := range opts.RawListenAddresses {
+		switch addr {
+		case "default":
+			addresses = append(addresses, DefaultListenAddresses...)
+		default:
+			addresses = append(addresses, addr)
+		}
+	}
+	return util.UniqueTrimmedStrings(addresses)
+}
+
+func (opts OptionsConfiguration) StunServers() []string {
+	var addresses []string
+	for _, addr := range opts.RawStunServers {
+		switch addr {
+		case "default":
+			defaultPrimaryAddresses := make([]string, len(DefaultPrimaryStunServers))
+			copy(defaultPrimaryAddresses, DefaultPrimaryStunServers)
+			rand.Shuffle(defaultPrimaryAddresses)
+			addresses = append(addresses, defaultPrimaryAddresses...)
+
+			defaultSecondaryAddresses := make([]string, len(DefaultSecondaryStunServers))
+			copy(defaultSecondaryAddresses, DefaultSecondaryStunServers)
+			rand.Shuffle(defaultSecondaryAddresses)
+			addresses = append(addresses, defaultSecondaryAddresses...)
+		default:
+			addresses = append(addresses, addr)
+		}
+	}
+
+	addresses = util.UniqueTrimmedStrings(addresses)
+
+	return addresses
+}
+
+func (opts OptionsConfiguration) GlobalDiscoveryServers() []string {
+	var servers []string
+	for _, srv := range opts.RawGlobalAnnServers {
+		switch srv {
+		case "default":
+			servers = append(servers, DefaultDiscoveryServers...)
+		case "default-v4":
+			servers = append(servers, DefaultDiscoveryServersV4...)
+		case "default-v6":
+			servers = append(servers, DefaultDiscoveryServersV6...)
+		default:
+			servers = append(servers, srv)
+		}
+	}
+	return util.UniqueTrimmedStrings(servers)
+}

lib/config/size.go

@@ -87,11 +87,26 @@ func CheckFreeSpace(req Size, usage fs.Usage) error {
 	if req.Percentage() {
 		freePct := (float64(usage.Free) / float64(usage.Total)) * 100
 		if freePct < val {
-			return fmt.Errorf("%f %% < %v", freePct, req)
+			return fmt.Errorf("%.1f %% < %v", freePct, req)
 		}
 	} else if float64(usage.Free) < val {
-		return fmt.Errorf("%v < %v", usage.Free, req)
+		return fmt.Errorf("%sB < %v", formatSI(usage.Free), req)
 	}
 
 	return nil
 }
+
+func formatSI(b int64) string {
+	switch {
+	case b < 1000:
+		return fmt.Sprintf("%d ", b)
+	case b < 1000*1000:
+		return fmt.Sprintf("%.1f K", float64(b)/1000)
+	case b < 1000*1000*1000:
+		return fmt.Sprintf("%.1f M", float64(b)/(1000*1000))
+	case b < 1000*1000*1000*1000:
+		return fmt.Sprintf("%.1f G", float64(b)/(1000*1000*1000))
+	default:
+		return fmt.Sprintf("%.1f T", float64(b)/(1000*1000*1000*1000))
+	}
+}

lib/config/size_test.go

@@ -91,3 +91,42 @@ func TestParseSize(t *testing.T) {
 		}
 	}
 }
+
+func TestFormatSI(t *testing.T) {
+	cases := []struct {
+		bytes  int64
+		result string
+	}{
+		{
+			bytes:  0,
+			result: "0 ", // space for unit
+		},
+		{
+			bytes:  999,
+			result: "999 ",
+		},
+		{
+			bytes:  1000,
+			result: "1.0 K",
+		},
+		{
+			bytes:  1023 * 1000,
+			result: "1.0 M",
+		},
+		{
+			bytes:  5 * 1000 * 1000 * 1000,
+			result: "5.0 G",
+		},
+		{
+			bytes:  50000 * 1000 * 1000 * 1000 * 1000,
+			result: "50000.0 T",
+		},
+	}
+
+	for _, tc := range cases {
+		res := formatSI(tc.bytes)
+		if res != tc.result {
+			t.Errorf("formatSI(%d) => %q, expected %q", tc.bytes, res, tc.result)
+		}
+	}
+}

lib/config/tuning.go

@@ -0,0 +1,47 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package config
+
+type Tuning int
+
+const (
+	// N.b. these constants must match those in lib/db.Tuning!
+	TuningAuto Tuning = iota // default is auto
+	TuningSmall
+	TuningLarge
+)
+
+func (t Tuning) String() string {
+	switch t {
+	case TuningAuto:
+		return "auto"
+	case TuningSmall:
+		return "small"
+	case TuningLarge:
+		return "large"
+	default:
+		return "unknown"
+	}
+}
+
+func (t Tuning) MarshalText() ([]byte, error) {
+	return []byte(t.String()), nil
+}
+
+func (t *Tuning) UnmarshalText(bs []byte) error {
+	switch string(bs) {
+	case "auto":
+		*t = TuningAuto
+	case "small":
+		*t = TuningSmall
+	case "large":
+		*t = TuningLarge
+	default:
+		*t = TuningAuto
+	}
+	return nil
+}

lib/config/tuning_test.go

@@ -0,0 +1,26 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package config_test
+
+import (
+	"testing"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/db/backend"
+)
+
+func TestTuningMatches(t *testing.T) {
+	if int(config.TuningAuto) != int(backend.TuningAuto) {
+		t.Error("mismatch for TuningAuto")
+	}
+	if int(config.TuningSmall) != int(backend.TuningSmall) {
+		t.Error("mismatch for TuningSmall")
+	}
+	if int(config.TuningLarge) != int(backend.TuningLarge) {
+		t.Error("mismatch for TuningLarge")
+	}
+}

lib/config/wrapper.go

@@ -14,9 +14,7 @@ import (
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/sync"
-	"github.com/syncthing/syncthing/lib/util"
 )
 
 // The Committer interface is implemented by objects that need to know about
@@ -87,18 +85,16 @@ type Wrapper interface {
 	IgnoredDevice(id protocol.DeviceID) bool
 	IgnoredFolder(device protocol.DeviceID, folder string) bool
 
-	ListenAddresses() []string
-	GlobalDiscoveryServers() []string
-	StunServers() []string
-
 	Subscribe(c Committer)
 	Unsubscribe(c Committer)
 }
 
 type wrapper struct {
-	cfg  Configuration
-	path string
+	cfg      Configuration
+	path     string
+	evLogger events.Logger
 
+	waiter    Waiter // Latest ongoing config change
 	deviceMap map[protocol.DeviceID]DeviceConfiguration
 	folderMap map[string]FolderConfiguration
 	subs      []Committer
@@ -107,44 +103,22 @@ type wrapper struct {
 	requiresRestart uint32 // an atomic bool
 }
 
-func (w *wrapper) StunServers() []string {
-	var addresses []string
-	for _, addr := range w.cfg.Options.StunServers {
-		switch addr {
-		case "default":
-			defaultPrimaryAddresses := make([]string, len(DefaultPrimaryStunServers))
-			copy(defaultPrimaryAddresses, DefaultPrimaryStunServers)
-			rand.Shuffle(defaultPrimaryAddresses)
-			addresses = append(addresses, defaultPrimaryAddresses...)
-
-			defaultSecondaryAddresses := make([]string, len(DefaultSecondaryStunServers))
-			copy(defaultSecondaryAddresses, DefaultSecondaryStunServers)
-			rand.Shuffle(defaultSecondaryAddresses)
-			addresses = append(addresses, defaultSecondaryAddresses...)
-		default:
-			addresses = append(addresses, addr)
-		}
-	}
-
-	addresses = util.UniqueTrimmedStrings(addresses)
-
-	return addresses
-}
-
 // Wrap wraps an existing Configuration structure and ties it to a file on
 // disk.
-func Wrap(path string, cfg Configuration) Wrapper {
+func Wrap(path string, cfg Configuration, evLogger events.Logger) Wrapper {
 	w := &wrapper{
-		cfg:  cfg,
-		path: path,
-		mut:  sync.NewMutex(),
+		cfg:      cfg,
+		path:     path,
+		evLogger: evLogger,
+		waiter:   noopWaiter{}, // Noop until first config change
+		mut:      sync.NewMutex(),
 	}
 	return w
 }
 
 // Load loads an existing file on disk and returns a new configuration
 // wrapper.
-func Load(path string, myID protocol.DeviceID) (Wrapper, error) {
+func Load(path string, myID protocol.DeviceID, evLogger events.Logger) (Wrapper, error) {
 	fd, err := os.Open(path)
 	if err != nil {
 		return nil, err
@@ -156,7 +130,7 @@ func Load(path string, myID protocol.DeviceID) (Wrapper, error) {
 		return nil, err
 	}
 
-	return Wrap(path, cfg), nil
+	return Wrap(path, cfg, evLogger), nil
 }
 
 func (w *wrapper) ConfigPath() string {
@@ -172,7 +146,8 @@ func (w *wrapper) Subscribe(c Committer) {
 }
 
 // Unsubscribe de-registers the given handler from any future calls to
-// configuration changes
+// configuration changes and only returns after a potential ongoing config
+// change is done.
 func (w *wrapper) Unsubscribe(c Committer) {
 	w.mut.Lock()
 	for i := range w.subs {
@@ -183,7 +158,11 @@ func (w *wrapper) Unsubscribe(c Committer) {
 			break
 		}
 	}
+	waiter := w.waiter
 	w.mut.Unlock()
+	// Waiting mustn't be done under lock, as the goroutines in notifyListener
+	// may dead-lock when trying to access lock on config read operations.
+	waiter.Wait()
 }
 
 // RawCopy returns a copy of the currently wrapped Configuration object.
@@ -219,7 +198,9 @@ func (w *wrapper) replaceLocked(to Configuration) (Waiter, error) {
 	w.deviceMap = nil
 	w.folderMap = nil
 
-	return w.notifyListeners(from.Copy(), to.Copy()), nil
+	w.waiter = w.notifyListeners(from.Copy(), to.Copy())
+
+	return w.waiter, nil
 }
 
 func (w *wrapper) notifyListeners(from, to Configuration) Waiter {
@@ -450,40 +431,10 @@ func (w *wrapper) Save() error {
 		return err
 	}
 
-	events.Default.Log(events.ConfigSaved, w.cfg)
+	w.evLogger.Log(events.ConfigSaved, w.cfg)
 	return nil
 }
 
-func (w *wrapper) GlobalDiscoveryServers() []string {
-	var servers []string
-	for _, srv := range w.Options().GlobalAnnServers {
-		switch srv {
-		case "default":
-			servers = append(servers, DefaultDiscoveryServers...)
-		case "default-v4":
-			servers = append(servers, DefaultDiscoveryServersV4...)
-		case "default-v6":
-			servers = append(servers, DefaultDiscoveryServersV6...)
-		default:
-			servers = append(servers, srv)
-		}
-	}
-	return util.UniqueTrimmedStrings(servers)
-}
-
-func (w *wrapper) ListenAddresses() []string {
-	var addresses []string
-	for _, addr := range w.Options().ListenAddresses {
-		switch addr {
-		case "default":
-			addresses = append(addresses, DefaultListenAddresses...)
-		default:
-			addresses = append(addresses, addr)
-		}
-	}
-	return util.UniqueTrimmedStrings(addresses)
-}
-
 func (w *wrapper) RequiresRestart() bool {
 	return atomic.LoadUint32(&w.requiresRestart) != 0
 }
@@ -501,8 +452,6 @@ func (w *wrapper) MyName() string {
 }
 
 func (w *wrapper) AddOrUpdatePendingDevice(device protocol.DeviceID, name, address string) {
-	defer w.Save()
-
 	w.mut.Lock()
 	defer w.mut.Unlock()
 
@@ -524,8 +473,6 @@ func (w *wrapper) AddOrUpdatePendingDevice(device protocol.DeviceID, name, addre
 }
 
 func (w *wrapper) AddOrUpdatePendingFolder(id, label string, device protocol.DeviceID) {
-	defer w.Save()
-
 	w.mut.Lock()
 	defer w.mut.Unlock()
 

lib/connections/debug.go

@@ -7,16 +7,9 @@
 package connections
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
 	l = logger.DefaultLogger.NewFacility("connections", "Connection handling")
 )
-
-func init() {
-	l.SetDebug("connections", strings.Contains(os.Getenv("STTRACE"), "connections") || os.Getenv("STTRACE") == "all")
-}

lib/connections/lan_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
 )
 
 func TestIsLANHost(t *testing.T) {
@@ -35,7 +36,7 @@ func TestIsLANHost(t *testing.T) {
 		Options: config.OptionsConfiguration{
 			AlwaysLocalNets: []string{"10.20.30.0/24"},
 		},
-	})
+	}, events.NoopLogger)
 	s := &service{cfg: cfg}
 
 	for _, tc := range cases {

lib/connections/limiter.go

@@ -32,9 +32,13 @@ type limiter struct {
 type waiter interface {
 	// This is the rate limiting operation
 	WaitN(ctx context.Context, n int) error
+	Limit() rate.Limit
 }
 
-const limiterBurstSize = 4 * 128 << 10
+const (
+	limiterBurstSize   = 4 * 128 << 10
+	maxSingleWriteSize = 8 << 10
+)
 
 func newLimiter(cfg config.Wrapper) *limiter {
 	l := &limiter{
@@ -186,19 +190,23 @@ func (lim *limiter) getLimiters(remoteID protocol.DeviceID, rw io.ReadWriter, is
 
 func (lim *limiter) newLimitedReaderLocked(remoteID protocol.DeviceID, r io.Reader, isLAN bool) io.Reader {
 	return &limitedReader{
-		reader:    r,
-		limitsLAN: &lim.limitsLAN,
-		waiter:    totalWaiter{lim.getReadLimiterLocked(remoteID), lim.read},
-		isLAN:     isLAN,
+		reader: r,
+		waiterHolder: waiterHolder{
+			waiter:    totalWaiter{lim.getReadLimiterLocked(remoteID), lim.read},
+			limitsLAN: &lim.limitsLAN,
+			isLAN:     isLAN,
+		},
 	}
 }
 
 func (lim *limiter) newLimitedWriterLocked(remoteID protocol.DeviceID, w io.Writer, isLAN bool) io.Writer {
 	return &limitedWriter{
-		writer:    w,
-		limitsLAN: &lim.limitsLAN,
-		waiter:    totalWaiter{lim.getWriteLimiterLocked(remoteID), lim.write},
-		isLAN:     isLAN,
+		writer: w,
+		waiterHolder: waiterHolder{
+			waiter:    totalWaiter{lim.getWriteLimiterLocked(remoteID), lim.write},
+			limitsLAN: &lim.limitsLAN,
+			isLAN:     isLAN,
+		},
 	}
 }
 
@@ -221,53 +229,87 @@ func getRateLimiter(m map[protocol.DeviceID]*rate.Limiter, deviceID protocol.Dev
 
 // limitedReader is a rate limited io.Reader
 type limitedReader struct {
-	reader    io.Reader
-	limitsLAN *atomicBool
-	waiter    waiter
-	isLAN     bool
+	reader io.Reader
+	waiterHolder
 }
 
 func (r *limitedReader) Read(buf []byte) (int, error) {
 	n, err := r.reader.Read(buf)
-	if !r.isLAN || r.limitsLAN.get() {
-		take(r.waiter, n)
+	if !r.unlimited() {
+		r.take(n)
 	}
 	return n, err
 }
 
 // limitedWriter is a rate limited io.Writer
 type limitedWriter struct {
-	writer    io.Writer
-	limitsLAN *atomicBool
-	waiter    waiter
-	isLAN     bool
+	writer io.Writer
+	waiterHolder
 }
 
 func (w *limitedWriter) Write(buf []byte) (int, error) {
-	if !w.isLAN || w.limitsLAN.get() {
-		take(w.waiter, len(buf))
+	if w.unlimited() {
+		return w.writer.Write(buf)
 	}
-	return w.writer.Write(buf)
+
+	// This does (potentially) multiple smaller writes in order to be less
+	// bursty with large writes and slow rates.
+	written := 0
+	for written < len(buf) {
+		toWrite := maxSingleWriteSize
+		if toWrite > len(buf)-written {
+			toWrite = len(buf) - written
+		}
+		w.take(toWrite)
+		n, err := w.writer.Write(buf[written : written+toWrite])
+		written += n
+		if err != nil {
+			return written, err
+		}
+	}
+
+	return written, nil
 }
 
-// take is a utility function to consume tokens from a overall rate.Limiter and deviceLimiter.
-// No call to WaitN can be larger than the limiter burst size so we split it up into
-// several calls when necessary.
-func take(waiter waiter, tokens int) {
+// waiterHolder is the common functionality around having and evaluating a
+// waiter, valid for both writers and readers
+type waiterHolder struct {
+	waiter    waiter
+	limitsLAN *atomicBool
+	isLAN     bool
+}
+
+// unlimited returns true if the waiter is not limiting the rate
+func (w waiterHolder) unlimited() bool {
+	if w.isLAN && !w.limitsLAN.get() {
+		return true
+	}
+	return w.waiter.Limit() == rate.Inf
+}
+
+// take is a utility function to consume tokens, because no call to WaitN
+// must be larger than the limiter burst size or it will hang.
+func (w waiterHolder) take(tokens int) {
+	// For writes we already split the buffer into smaller operations so those
+	// will always end up in the fast path below. For reads, however, we don't
+	// control the size of the incoming buffer and don't split the calls
+	// into the lower level reads so we might get a large amount of data and
+	// end up in the loop further down.
+
 	if tokens < limiterBurstSize {
-		// This is the by far more common case so we get it out of the way
-		// early.
-		waiter.WaitN(context.TODO(), tokens)
+		// Fast path. We won't get an error from WaitN as we don't pass a
+		// context with a deadline.
+		_ = w.waiter.WaitN(context.TODO(), tokens)
 		return
 	}
 
 	for tokens > 0 {
 		// Consume limiterBurstSize tokens at a time until we're done.
 		if tokens > limiterBurstSize {
-			waiter.WaitN(context.TODO(), limiterBurstSize)
+			_ = w.waiter.WaitN(context.TODO(), limiterBurstSize)
 			tokens -= limiterBurstSize
 		} else {
-			waiter.WaitN(context.TODO(), tokens)
+			_ = w.waiter.WaitN(context.TODO(), tokens)
 			tokens = 0
 		}
 	}
@@ -300,3 +342,13 @@ func (tw totalWaiter) WaitN(ctx context.Context, n int) error {
 	}
 	return nil
 }
+
+func (tw totalWaiter) Limit() rate.Limit {
+	min := rate.Inf
+	for _, w := range tw {
+		if l := w.Limit(); l < min {
+			min = l
+		}
+	}
+	return min
+}

lib/connections/limiter_test.go

@@ -7,11 +7,16 @@
 package connections
 
 import (
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"golang.org/x/time/rate"
+	"bytes"
+	crand "crypto/rand"
+	"io"
 	"math/rand"
 	"testing"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"golang.org/x/time/rate"
 )
 
 var device1, device2, device3, device4 protocol.DeviceID
@@ -25,7 +30,7 @@ func init() {
 }
 
 func initConfig() config.Wrapper {
-	cfg := config.Wrap("/dev/null", config.New(device1))
+	cfg := config.Wrap("/dev/null", config.New(device1), events.NoopLogger)
 	dev1Conf = config.NewDeviceConfiguration(device1, "device1")
 	dev2Conf = config.NewDeviceConfiguration(device2, "device2")
 	dev3Conf = config.NewDeviceConfiguration(device3, "device3")
@@ -184,6 +189,151 @@ func TestAddAndRemove(t *testing.T) {
 	checkActualAndExpected(t, actualR, actualW, expectedR, expectedW)
 }
 
+func TestLimitedWriterWrite(t *testing.T) {
+	// Check that the limited writer writes the correct data in the correct manner.
+
+	// A buffer with random data that is larger than the write size and not
+	// a precise multiple either.
+	src := make([]byte, int(12.5*maxSingleWriteSize))
+	if _, err := crand.Reader.Read(src); err != nil {
+		t.Fatal(err)
+	}
+
+	// Write it to the destination using a limited writer, with a wrapper to
+	// count the write calls. The defaults on the limited writer should mean
+	// it is used (and doesn't take the fast path). In practice the limiter
+	// won't delay the test as the burst size is large enough to accommodate
+	// regardless of the rate.
+	dst := new(bytes.Buffer)
+	cw := &countingWriter{w: dst}
+	lw := &limitedWriter{
+		writer: cw,
+		waiterHolder: waiterHolder{
+			waiter:    rate.NewLimiter(rate.Limit(42), limiterBurstSize),
+			limitsLAN: new(atomicBool),
+			isLAN:     false, // enables limiting
+		},
+	}
+	if _, err := io.Copy(lw, bytes.NewReader(src)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify there were lots of writes and that the end result is identical.
+	if cw.writeCount != 13 {
+		t.Error("expected lots of smaller writes, but not too many")
+	}
+	if !bytes.Equal(src, dst.Bytes()) {
+		t.Error("results should be equal")
+	}
+
+	// Write it to the destination using a limited writer, with a wrapper to
+	// count the write calls. Now we make sure the fast path is used.
+	dst = new(bytes.Buffer)
+	cw = &countingWriter{w: dst}
+	lw = &limitedWriter{
+		writer: cw,
+		waiterHolder: waiterHolder{
+			waiter:    rate.NewLimiter(rate.Limit(42), limiterBurstSize),
+			limitsLAN: new(atomicBool),
+			isLAN:     true, // disables limiting
+		},
+	}
+	if _, err := io.Copy(lw, bytes.NewReader(src)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify there were a single write and that the end result is identical.
+	if cw.writeCount != 1 {
+		t.Error("expected just the one write")
+	}
+	if !bytes.Equal(src, dst.Bytes()) {
+		t.Error("results should be equal")
+	}
+
+	// Once more, but making sure the fast path is used for an unlimited
+	// rate, with multiple unlimited raters even (global and per-device).
+	dst = new(bytes.Buffer)
+	cw = &countingWriter{w: dst}
+	lw = &limitedWriter{
+		writer: cw,
+		waiterHolder: waiterHolder{
+			waiter:    totalWaiter{rate.NewLimiter(rate.Inf, limiterBurstSize), rate.NewLimiter(rate.Inf, limiterBurstSize)},
+			limitsLAN: new(atomicBool),
+			isLAN:     false, // enables limiting
+		},
+	}
+	if _, err := io.Copy(lw, bytes.NewReader(src)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify there were a single write and that the end result is identical.
+	if cw.writeCount != 1 {
+		t.Error("expected just the one write")
+	}
+	if !bytes.Equal(src, dst.Bytes()) {
+		t.Error("results should be equal")
+	}
+
+	// Once more, but making sure we *don't* take the fast path when there
+	// is a combo of limited and unlimited writers.
+	dst = new(bytes.Buffer)
+	cw = &countingWriter{w: dst}
+	lw = &limitedWriter{
+		writer: cw,
+		waiterHolder: waiterHolder{
+			waiter: totalWaiter{
+				rate.NewLimiter(rate.Inf, limiterBurstSize),
+				rate.NewLimiter(rate.Limit(42), limiterBurstSize),
+				rate.NewLimiter(rate.Inf, limiterBurstSize),
+			},
+			limitsLAN: new(atomicBool),
+			isLAN:     false, // enables limiting
+		},
+	}
+	if _, err := io.Copy(lw, bytes.NewReader(src)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify there were lots of writes and that the end result is identical.
+	if cw.writeCount != 13 {
+		t.Error("expected just the one write")
+	}
+	if !bytes.Equal(src, dst.Bytes()) {
+		t.Error("results should be equal")
+	}
+}
+
+func TestTotalWaiterLimit(t *testing.T) {
+	cases := []struct {
+		w waiter
+		r rate.Limit
+	}{
+		{
+			totalWaiter{},
+			rate.Inf,
+		},
+		{
+			totalWaiter{rate.NewLimiter(rate.Inf, 42)},
+			rate.Inf,
+		},
+		{
+			totalWaiter{rate.NewLimiter(rate.Inf, 42), rate.NewLimiter(rate.Inf, 42)},
+			rate.Inf,
+		},
+		{
+			totalWaiter{rate.NewLimiter(rate.Inf, 42), rate.NewLimiter(rate.Limit(12), 42), rate.NewLimiter(rate.Limit(15), 42)},
+			rate.Limit(12),
+		},
+	}
+
+	for _, tc := range cases {
+		l := tc.w.Limit()
+		if l != tc.r {
+			t.Error("incorrect limit returned")
+		}
+	}
+}
+
 func checkActualAndExpected(t *testing.T, actualR, actualW, expectedR, expectedW map[protocol.DeviceID]*rate.Limiter) {
 	t.Helper()
 	if len(expectedW) != len(actualW) || len(expectedR) != len(actualR) {
@@ -203,3 +353,13 @@ func checkActualAndExpected(t *testing.T, actualR, actualW, expectedR, expectedW
 		}
 	}
 }
+
+type countingWriter struct {
+	w          io.Writer
+	writeCount int
+}
+
+func (w *countingWriter) Write(data []byte) (int, error) {
+	w.writeCount++
+	return w.w.Write(data)
+}

lib/connections/quic_dial.go

@@ -16,13 +16,20 @@ import (
 	"time"
 
 	"github.com/lucas-clemente/quic-go"
+	"github.com/pkg/errors"
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/connections/registry"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-const quicPriority = 100
+const (
+	quicPriority = 100
+
+	// The timeout for connecting, accepting and creating the various
+	// streams.
+	quicOperationTimeout = 10 * time.Second
+)
 
 func init() {
 	factory := &quicDialerFactory{}
@@ -32,11 +39,10 @@ func init() {
 }
 
 type quicDialer struct {
-	cfg    config.Wrapper
-	tlsCfg *tls.Config
+	commonDialer
 }
 
-func (d *quicDialer) Dial(id protocol.DeviceID, uri *url.URL) (internalConn, error) {
+func (d *quicDialer) Dial(ctx context.Context, _ protocol.DeviceID, uri *url.URL) (internalConn, error) {
 	uri = fixupPort(uri, config.DefaultQUICPort)
 
 	addr, err := net.ResolveUDPAddr("udp", uri.Host)
@@ -60,57 +66,40 @@ func (d *quicDialer) Dial(id protocol.DeviceID, uri *url.URL) (internalConn, err
 		}
 	}
 
-	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
+	ctx, cancel := context.WithTimeout(ctx, quicOperationTimeout)
+	defer cancel()
+
 	session, err := quic.DialContext(ctx, conn, addr, uri.Host, d.tlsCfg, quicConfig)
 	if err != nil {
 		if createdConn != nil {
 			_ = createdConn.Close()
 		}
-		return internalConn{}, err
+		return internalConn{}, errors.Wrap(err, "dial")
 	}
 
-	// OpenStreamSync is blocks, but we want to make sure the connection is usable
-	// before we start killing off other connections, so do the dance.
-	ok := make(chan struct{})
-	go func() {
-		select {
-		case <-ok:
-			return
-		case <-time.After(10 * time.Second):
-			l.Debugln("timed out waiting for OpenStream on", session.RemoteAddr())
-			// This will unblock OpenStreamSync
-			_ = session.Close()
-		}
-	}()
-
-	stream, err := session.OpenStreamSync()
-	close(ok)
+	stream, err := session.OpenStreamSync(ctx)
 	if err != nil {
 		// It's ok to close these, this does not close the underlying packetConn.
 		_ = session.Close()
 		if createdConn != nil {
 			_ = createdConn.Close()
 		}
-		return internalConn{}, err
+		return internalConn{}, errors.Wrap(err, "open stream")
 	}
 
 	return internalConn{&quicTlsConn{session, stream, createdConn}, connTypeQUICClient, quicPriority}, nil
 }
 
-func (d *quicDialer) RedialFrequency() time.Duration {
-	return time.Duration(d.cfg.Options().ReconnectIntervalS) * time.Second
-}
-
 type quicDialerFactory struct {
 	cfg    config.Wrapper
 	tlsCfg *tls.Config
 }
 
-func (quicDialerFactory) New(cfg config.Wrapper, tlsCfg *tls.Config) genericDialer {
-	return &quicDialer{
-		cfg:    cfg,
-		tlsCfg: tlsCfg,
-	}
+func (quicDialerFactory) New(opts config.OptionsConfiguration, tlsCfg *tls.Config) genericDialer {
+	return &quicDialer{commonDialer{
+		reconnectInterval: time.Duration(opts.ReconnectIntervalS) * time.Second,
+		tlsCfg:            tlsCfg,
+	}}
 }
 
 func (quicDialerFactory) Priority() int {

lib/connections/quic_listen.go

@@ -9,6 +9,7 @@
 package connections
 
 import (
+	"context"
 	"crypto/tls"
 	"net"
 	"net/url"
@@ -77,7 +78,7 @@ func (t *quicListener) OnExternalAddressChanged(address *stun.Host, via string)
 	}
 }
 
-func (t *quicListener) serve(stop chan struct{}) error {
+func (t *quicListener) serve(ctx context.Context) error {
 	network := strings.Replace(t.uri.Scheme, "quic", "udp", -1)
 
 	packetConn, err := net.ListenPacket(network, t.uri.Host)
@@ -101,57 +102,49 @@ func (t *quicListener) serve(stop chan struct{}) error {
 		l.Infoln("Listen (BEP/quic):", err)
 		return err
 	}
+	defer listener.Close()
 
 	l.Infof("QUIC listener (%v) starting", packetConn.LocalAddr())
 	defer l.Infof("QUIC listener (%v) shutting down", packetConn.LocalAddr())
 
-	// Accept is forever, so handle stops externally.
-	go func() {
-		select {
-		case <-stop:
-			_ = listener.Close()
-		}
-	}()
+	acceptFailures := 0
+	const maxAcceptFailures = 10
 
 	for {
-		// Blocks forever, see https://github.com/lucas-clemente/quic-go/issues/1915
-		session, err := listener.Accept()
-
 		select {
-		case <-stop:
-			if err == nil {
-				_ = session.Close()
-			}
+		case <-ctx.Done():
 			return nil
 		default:
 		}
-		if err != nil {
-			if err, ok := err.(net.Error); !ok || !err.Timeout() {
-				l.Warnln("Listen (BEP/quic): Accepting connection:", err)
+
+		session, err := listener.Accept(ctx)
+		if err == context.Canceled {
+			return nil
+		} else if err != nil {
+			l.Infoln("Listen (BEP/quic): Accepting connection:", err)
+
+			acceptFailures++
+			if acceptFailures > maxAcceptFailures {
+				// Return to restart the listener, because something
+				// seems permanently damaged.
+				return err
 			}
+
+			// Slightly increased delay for each failure.
+			time.Sleep(time.Duration(acceptFailures) * time.Second)
+
 			continue
 		}
 
+		acceptFailures = 0
+
 		l.Debugln("connect from", session.RemoteAddr())
 
-		// Accept blocks forever, give it 10s to do it's thing.
-		ok := make(chan struct{})
-		go func() {
-			select {
-			case <-ok:
-				return
-			case <-stop:
-				_ = session.Close()
-			case <-time.After(10 * time.Second):
-				l.Debugln("timed out waiting for AcceptStream on", session.RemoteAddr())
-				_ = session.Close()
-			}
-		}()
-
-		stream, err := session.AcceptStream()
-		close(ok)
+		streamCtx, cancel := context.WithTimeout(ctx, quicOperationTimeout)
+		stream, err := session.AcceptStream(streamCtx)
+		cancel()
 		if err != nil {
-			l.Debugln("failed to accept stream from", session.RemoteAddr(), err.Error())
+			l.Debugf("failed to accept stream from %s: %v", session.RemoteAddr(), err)
 			_ = session.Close()
 			continue
 		}
@@ -208,7 +201,7 @@ func (f *quicListenerFactory) New(uri *url.URL, cfg config.Wrapper, tlsCfg *tls.
 		conns:   conns,
 		factory: f,
 	}
-	l.ServiceWithError = util.AsServiceWithError(l.serve)
+	l.ServiceWithError = util.AsServiceWithError(l.serve, l.String())
 	l.nat.Store(stun.NATUnknown)
 	return l
 }

lib/connections/relay_dial.go

@@ -7,6 +7,7 @@
 package connections
 
 import (
+	"context"
 	"crypto/tls"
 	"net/url"
 	"time"
@@ -24,17 +25,16 @@ func init() {
 }
 
 type relayDialer struct {
-	cfg    config.Wrapper
-	tlsCfg *tls.Config
+	commonDialer
 }
 
-func (d *relayDialer) Dial(id protocol.DeviceID, uri *url.URL) (internalConn, error) {
-	inv, err := client.GetInvitationFromRelay(uri, id, d.tlsCfg.Certificates, 10*time.Second)
+func (d *relayDialer) Dial(ctx context.Context, id protocol.DeviceID, uri *url.URL) (internalConn, error) {
+	inv, err := client.GetInvitationFromRelay(ctx, uri, id, d.tlsCfg.Certificates, 10*time.Second)
 	if err != nil {
 		return internalConn{}, err
 	}
 
-	conn, err := client.JoinSession(inv)
+	conn, err := client.JoinSession(ctx, inv)
 	if err != nil {
 		return internalConn{}, err
 	}
@@ -45,7 +45,7 @@ func (d *relayDialer) Dial(id protocol.DeviceID, uri *url.URL) (internalConn, er
 		return internalConn{}, err
 	}
 
-	err = dialer.SetTrafficClass(conn, d.cfg.Options().TrafficClass)
+	err = dialer.SetTrafficClass(conn, d.trafficClass)
 	if err != nil {
 		l.Debugln("Dial (BEP/relay): setting traffic class:", err)
 	}
@@ -66,17 +66,14 @@ func (d *relayDialer) Dial(id protocol.DeviceID, uri *url.URL) (internalConn, er
 	return internalConn{tc, connTypeRelayClient, relayPriority}, nil
 }
 
-func (d *relayDialer) RedialFrequency() time.Duration {
-	return time.Duration(d.cfg.Options().RelayReconnectIntervalM) * time.Minute
-}
-
 type relayDialerFactory struct{}
 
-func (relayDialerFactory) New(cfg config.Wrapper, tlsCfg *tls.Config) genericDialer {
-	return &relayDialer{
-		cfg:    cfg,
-		tlsCfg: tlsCfg,
-	}
+func (relayDialerFactory) New(opts config.OptionsConfiguration, tlsCfg *tls.Config) genericDialer {
+	return &relayDialer{commonDialer{
+		trafficClass:      opts.TrafficClass,
+		reconnectInterval: time.Duration(opts.RelayReconnectIntervalM) * time.Minute,
+		tlsCfg:            tlsCfg,
+	}}
 }
 
 func (relayDialerFactory) Priority() int {

lib/connections/relay_listen.go

@@ -7,15 +7,19 @@
 package connections
 
 import (
+	"context"
 	"crypto/tls"
 	"net/url"
 	"sync"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/dialer"
 	"github.com/syncthing/syncthing/lib/nat"
 	"github.com/syncthing/syncthing/lib/relay/client"
+	"github.com/syncthing/syncthing/lib/util"
 )
 
 func init() {
@@ -26,6 +30,7 @@ func init() {
 }
 
 type relayListener struct {
+	util.ServiceWithError
 	onAddressesChangedNotifier
 
 	uri     *url.URL
@@ -34,30 +39,22 @@ type relayListener struct {
 	conns   chan internalConn
 	factory listenerFactory
 
-	err    error
 	client client.RelayClient
 	mut    sync.RWMutex
 }
 
-func (t *relayListener) Serve() {
-	t.mut.Lock()
-	t.err = nil
-	t.mut.Unlock()
-
+func (t *relayListener) serve(ctx context.Context) error {
 	clnt, err := client.NewClient(t.uri, t.tlsCfg.Certificates, nil, 10*time.Second)
-	invitations := clnt.Invitations()
 	if err != nil {
-		t.mut.Lock()
-		t.err = err
-		t.mut.Unlock()
-		l.Warnln("Listen (BEP/relay):", err)
-		return
+		l.Infoln("Listen (BEP/relay):", err)
+		return err
 	}
-
-	go clnt.Serve()
+	invitations := clnt.Invitations()
 
 	t.mut.Lock()
 	t.client = clnt
+	go clnt.Serve()
+	defer clnt.Stop()
 	t.mut.Unlock()
 
 	oldURI := clnt.URI()
@@ -69,12 +66,17 @@ func (t *relayListener) Serve() {
 		select {
 		case inv, ok := <-invitations:
 			if !ok {
-				return
+				if err := clnt.Error(); err != nil {
+					l.Infoln("Listen (BEP/relay):", err)
+				}
+				return err
 			}
 
-			conn, err := client.JoinSession(inv)
+			conn, err := client.JoinSession(ctx, inv)
 			if err != nil {
-				l.Infoln("Listen (BEP/relay): joining session:", err)
+				if errors.Cause(err) != context.Canceled {
+					l.Infoln("Listen (BEP/relay): joining session:", err)
+				}
 				continue
 			}
 
@@ -114,18 +116,13 @@ func (t *relayListener) Serve() {
 				oldURI = currentURI
 				t.notifyAddressesChanged(t)
 			}
+
+		case <-ctx.Done():
+			return nil
 		}
 	}
 }
 
-func (t *relayListener) Stop() {
-	t.mut.RLock()
-	if t.client != nil {
-		t.client.Stop()
-	}
-	t.mut.RUnlock()
-}
-
 func (t *relayListener) URI() *url.URL {
 	return t.uri
 }
@@ -152,18 +149,16 @@ func (t *relayListener) LANAddresses() []*url.URL {
 }
 
 func (t *relayListener) Error() error {
-	t.mut.RLock()
-	err := t.err
-	var cerr error
-	if t.client != nil {
-		cerr = t.client.Error()
-	}
-	t.mut.RUnlock()
-
+	err := t.ServiceWithError.Error()
 	if err != nil {
 		return err
 	}
-	return cerr
+	t.mut.RLock()
+	defer t.mut.RUnlock()
+	if t.client != nil {
+		return t.client.Error()
+	}
+	return nil
 }
 
 func (t *relayListener) Factory() listenerFactory {
@@ -181,13 +176,15 @@ func (t *relayListener) NATType() string {
 type relayListenerFactory struct{}
 
 func (f *relayListenerFactory) New(uri *url.URL, cfg config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {
-	return &relayListener{
+	t := &relayListener{
 		uri:     uri,
 		cfg:     cfg,
 		tlsCfg:  tlsCfg,
 		conns:   conns,
 		factory: f,
 	}
+	t.ServiceWithError = util.AsServiceWithError(t.serve, t.String())
+	return t
 }
 
 func (relayListenerFactory) Valid(cfg config.Configuration) error {

lib/connections/service.go

@@ -7,8 +7,8 @@
 package connections
 
 import (
+	"context"
 	"crypto/tls"
-	"errors"
 	"fmt"
 	"net"
 	"net/url"
@@ -30,6 +30,7 @@ import (
 	_ "github.com/syncthing/syncthing/lib/pmp"
 	_ "github.com/syncthing/syncthing/lib/upnp"
 
+	"github.com/pkg/errors"
 	"github.com/thejerf/suture"
 	"golang.org/x/time/rate"
 )
@@ -83,13 +84,14 @@ var tlsCipherSuiteNames = map[uint16]string{
 
 var tlsVersionNames = map[uint16]string{
 	tls.VersionTLS12: "TLS1.2",
-	772:              "TLS1.3", // tls.VersionTLS13 constant available in Go 1.12+
+	tls.VersionTLS13: "TLS1.3",
 }
 
 // Service listens and dials all configured unconnected devices, via supported
 // dialers. Successful connections are handed to the model.
 type Service interface {
 	suture.Service
+	discover.AddressLister
 	ListenerStatus() map[string]ListenerStatusEntry
 	ConnectionStatus() map[string]ConnectionStatusEntry
 	NATType() string
@@ -119,6 +121,7 @@ type service struct {
 	limiter              *limiter
 	natService           *nat.Service
 	natServiceToken      *suture.ServiceToken
+	evLogger             events.Logger
 
 	listenersMut       sync.RWMutex
 	listeners          map[string]genericListener
@@ -129,9 +132,7 @@ type service struct {
 	connectionStatus    map[string]ConnectionStatusEntry // address -> latest error/status
 }
 
-func NewService(cfg config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *tls.Config, discoverer discover.Finder,
-	bepProtocolName string, tlsDefaultCommonName string) *service {
-
+func NewService(cfg config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *tls.Config, discoverer discover.Finder, bepProtocolName string, tlsDefaultCommonName string, evLogger events.Logger) Service {
 	service := &service{
 		Supervisor: suture.New("connections.Service", suture.Spec{
 			Log: func(line string) {
@@ -149,6 +150,7 @@ func NewService(cfg config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *t
 		tlsDefaultCommonName: tlsDefaultCommonName,
 		limiter:              newLimiter(cfg),
 		natService:           nat.NewService(myID, cfg),
+		evLogger:             evLogger,
 
 		listenersMut:   sync.NewRWMutex(),
 		listeners:      make(map[string]genericListener),
@@ -184,18 +186,24 @@ func NewService(cfg config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *t
 	// the common handling regardless of whether the connection was
 	// incoming or outgoing.
 
-	service.Add(util.AsService(service.connect))
-	service.Add(util.AsService(service.handle))
+	service.Add(util.AsService(service.connect, fmt.Sprintf("%s/connect", service)))
+	service.Add(util.AsService(service.handle, fmt.Sprintf("%s/handle", service)))
 	service.Add(service.listenerSupervisor)
 
 	return service
 }
 
-func (s *service) handle(stop chan struct{}) {
+func (s *service) Stop() {
+	s.cfg.Unsubscribe(s.limiter)
+	s.cfg.Unsubscribe(s)
+	s.Supervisor.Stop()
+}
+
+func (s *service) handle(ctx context.Context) {
 	var c internalConn
 	for {
 		select {
-		case <-stop:
+		case <-ctx.Done():
 			return
 		case c = <-s.conns:
 		}
@@ -231,7 +239,7 @@ func (s *service) handle(stop chan struct{}) {
 			continue
 		}
 
-		c.SetDeadline(time.Now().Add(20 * time.Second))
+		_ = c.SetDeadline(time.Now().Add(20 * time.Second))
 		hello, err := protocol.ExchangeHello(c, s.model.GetHello(remoteID))
 		if err != nil {
 			if protocol.IsVersionMismatch(err) {
@@ -255,7 +263,7 @@ func (s *service) handle(stop chan struct{}) {
 			c.Close()
 			continue
 		}
-		c.SetDeadline(time.Time{})
+		_ = c.SetDeadline(time.Time{})
 
 		// The Model will return an error for devices that we don't want to
 		// have a connection with for whatever reason, for example unknown devices.
@@ -323,7 +331,7 @@ func (s *service) handle(stop chan struct{}) {
 	}
 }
 
-func (s *service) connect(stop chan struct{}) {
+func (s *service) connect(ctx context.Context) {
 	nextDial := make(map[string]time.Time)
 
 	// Used as delay for the first few connection attempts, increases
@@ -440,7 +448,7 @@ func (s *service) connect(stop chan struct{}) {
 					continue
 				}
 
-				dialer := dialerFactory.New(s.cfg, s.tlsCfg)
+				dialer := dialerFactory.New(s.cfg.Options(), s.tlsCfg)
 				nextDial[nextDialKey] = now.Add(dialer.RedialFrequency())
 
 				// For LAN addresses, increase the priority so that we
@@ -461,7 +469,7 @@ func (s *service) connect(stop chan struct{}) {
 				})
 			}
 
-			conn, ok := s.dialParallel(deviceCfg.DeviceID, dialTargets)
+			conn, ok := s.dialParallel(ctx, deviceCfg.DeviceID, dialTargets)
 			if ok {
 				s.conns <- conn
 			}
@@ -479,7 +487,7 @@ func (s *service) connect(stop chan struct{}) {
 
 		select {
 		case <-time.After(sleep):
-		case <-stop:
+		case <-ctx.Done():
 			return
 		}
 	}
@@ -553,7 +561,7 @@ func (s *service) createListener(factory listenerFactory, uri *url.URL) bool {
 }
 
 func (s *service) logListenAddressesChangedEvent(l genericListener) {
-	events.Default.Log(events.ListenAddressesChanged, map[string]interface{}{
+	s.evLogger.Log(events.ListenAddressesChanged, map[string]interface{}{
 		"address": l.URI(),
 		"lan":     l.LANAddresses(),
 		"wan":     l.WANAddresses(),
@@ -580,7 +588,7 @@ func (s *service) CommitConfiguration(from, to config.Configuration) bool {
 
 	s.listenersMut.Lock()
 	seen := make(map[string]struct{})
-	for _, addr := range config.Wrap("", to).ListenAddresses() {
+	for _, addr := range to.Options.ListenAddresses() {
 		if addr == "" {
 			// We can get an empty address if there is an empty listener
 			// element in the config, indicating no listeners should be
@@ -699,6 +707,10 @@ func (s *service) ConnectionStatus() map[string]ConnectionStatusEntry {
 }
 
 func (s *service) setConnectionStatus(address string, err error) {
+	if errors.Cause(err) != context.Canceled {
+		return
+	}
+
 	status := ConnectionStatusEntry{When: time.Now().UTC().Truncate(time.Second)}
 	if err != nil {
 		errStr := err.Error()
@@ -826,7 +838,7 @@ func IsAllowedNetwork(host string, allowed []string) bool {
 	return false
 }
 
-func (s *service) dialParallel(deviceID protocol.DeviceID, dialTargets []dialTarget) (internalConn, bool) {
+func (s *service) dialParallel(ctx context.Context, deviceID protocol.DeviceID, dialTargets []dialTarget) (internalConn, bool) {
 	// Group targets into buckets by priority
 	dialTargetBuckets := make(map[int][]dialTarget, len(dialTargets))
 	for _, tgt := range dialTargets {
@@ -849,9 +861,16 @@ func (s *service) dialParallel(deviceID protocol.DeviceID, dialTargets []dialTar
 		for _, tgt := range tgts {
 			wg.Add(1)
 			go func(tgt dialTarget) {
-				conn, err := tgt.Dial()
-				s.setConnectionStatus(tgt.addr, err)
+				conn, err := tgt.Dial(ctx)
 				if err == nil {
+					// Closes the connection on error
+					err = s.validateIdentity(conn, deviceID)
+				}
+				s.setConnectionStatus(tgt.addr, err)
+				if err != nil {
+					l.Debugln("dialing", deviceID, tgt.uri, "error:", err)
+				} else {
+					l.Debugln("dialing", deviceID, tgt.uri, "success:", conn)
 					res <- conn
 				}
 				wg.Done()
@@ -884,3 +903,36 @@ func (s *service) dialParallel(deviceID protocol.DeviceID, dialTargets []dialTar
 	}
 	return internalConn{}, false
 }
+
+func (s *service) validateIdentity(c internalConn, expectedID protocol.DeviceID) error {
+	cs := c.ConnectionState()
+
+	// We should have received exactly one certificate from the other
+	// side. If we didn't, they don't have a device ID and we drop the
+	// connection.
+	certs := cs.PeerCertificates
+	if cl := len(certs); cl != 1 {
+		l.Infof("Got peer certificate list of length %d != 1 from peer at %s; protocol error", cl, c)
+		c.Close()
+		return fmt.Errorf("expected 1 certificate, got %d", cl)
+	}
+	remoteCert := certs[0]
+	remoteID := protocol.NewDeviceID(remoteCert.Raw)
+
+	// The device ID should not be that of ourselves. It can happen
+	// though, especially in the presence of NAT hairpinning, multiple
+	// clients between the same NAT gateway, and global discovery.
+	if remoteID == s.myID {
+		l.Infof("Connected to myself (%s) at %s - should not happen", remoteID, c)
+		c.Close()
+		return fmt.Errorf("connected to self")
+	}
+
+	// We should see the expected device ID
+	if !remoteID.Equals(expectedID) {
+		c.Close()
+		return fmt.Errorf("unexpected device id, expected %s got %s", expectedID, remoteID)
+	}
+
+	return nil
+}

lib/connections/structs.go

@@ -7,6 +7,7 @@
 package connections
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"io"
@@ -146,15 +147,25 @@ func (c internalConn) String() string {
 }
 
 type dialerFactory interface {
-	New(config.Wrapper, *tls.Config) genericDialer
+	New(config.OptionsConfiguration, *tls.Config) genericDialer
 	Priority() int
 	AlwaysWAN() bool
 	Valid(config.Configuration) error
 	String() string
 }
 
+type commonDialer struct {
+	trafficClass      int
+	reconnectInterval time.Duration
+	tlsCfg            *tls.Config
+}
+
+func (d *commonDialer) RedialFrequency() time.Duration {
+	return d.reconnectInterval
+}
+
 type genericDialer interface {
-	Dial(protocol.DeviceID, *url.URL) (internalConn, error)
+	Dial(context.Context, protocol.DeviceID, *url.URL) (internalConn, error)
 	RedialFrequency() time.Duration
 }
 
@@ -213,13 +224,7 @@ type dialTarget struct {
 	deviceID protocol.DeviceID
 }
 
-func (t dialTarget) Dial() (internalConn, error) {
+func (t dialTarget) Dial(ctx context.Context) (internalConn, error) {
 	l.Debugln("dialing", t.deviceID, t.uri, "prio", t.priority)
-	conn, err := t.dialer.Dial(t.deviceID, t.uri)
-	if err != nil {
-		l.Debugln("dialing", t.deviceID, t.uri, "error:", err)
-	} else {
-		l.Debugln("dialing", t.deviceID, t.uri, "success:", conn)
-	}
-	return conn, err
+	return t.dialer.Dial(ctx, t.deviceID, t.uri)
 }

lib/connections/tcp_dial.go

@@ -7,6 +7,7 @@
 package connections
 
 import (
+	"context"
 	"crypto/tls"
 	"net/url"
 	"time"
@@ -26,14 +27,15 @@ func init() {
 }
 
 type tcpDialer struct {
-	cfg    config.Wrapper
-	tlsCfg *tls.Config
+	commonDialer
 }
 
-func (d *tcpDialer) Dial(id protocol.DeviceID, uri *url.URL) (internalConn, error) {
+func (d *tcpDialer) Dial(ctx context.Context, _ protocol.DeviceID, uri *url.URL) (internalConn, error) {
 	uri = fixupPort(uri, config.DefaultTCPPort)
 
-	conn, err := dialer.DialTimeout(uri.Scheme, uri.Host, 10*time.Second)
+	timeoutCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	conn, err := dialer.DialContext(timeoutCtx, uri.Scheme, uri.Host)
 	if err != nil {
 		return internalConn{}, err
 	}
@@ -43,7 +45,7 @@ func (d *tcpDialer) Dial(id protocol.DeviceID, uri *url.URL) (internalConn, erro
 		l.Debugln("Dial (BEP/tcp): setting tcp options:", err)
 	}
 
-	err = dialer.SetTrafficClass(conn, d.cfg.Options().TrafficClass)
+	err = dialer.SetTrafficClass(conn, d.trafficClass)
 	if err != nil {
 		l.Debugln("Dial (BEP/tcp): setting traffic class:", err)
 	}
@@ -58,17 +60,14 @@ func (d *tcpDialer) Dial(id protocol.DeviceID, uri *url.URL) (internalConn, erro
 	return internalConn{tc, connTypeTCPClient, tcpPriority}, nil
 }
 
-func (d *tcpDialer) RedialFrequency() time.Duration {
-	return time.Duration(d.cfg.Options().ReconnectIntervalS) * time.Second
-}
-
 type tcpDialerFactory struct{}
 
-func (tcpDialerFactory) New(cfg config.Wrapper, tlsCfg *tls.Config) genericDialer {
-	return &tcpDialer{
-		cfg:    cfg,
-		tlsCfg: tlsCfg,
-	}
+func (tcpDialerFactory) New(opts config.OptionsConfiguration, tlsCfg *tls.Config) genericDialer {
+	return &tcpDialer{commonDialer{
+		trafficClass:      opts.TrafficClass,
+		reconnectInterval: time.Duration(opts.ReconnectIntervalS) * time.Second,
+		tlsCfg:            tlsCfg,
+	}}
 }
 
 func (tcpDialerFactory) Priority() int {

lib/connections/tcp_listen.go

@@ -7,6 +7,7 @@
 package connections
 
 import (
+	"context"
 	"crypto/tls"
 	"net"
 	"net/url"
@@ -42,7 +43,7 @@ type tcpListener struct {
 	mut sync.RWMutex
 }
 
-func (t *tcpListener) serve(stop chan struct{}) error {
+func (t *tcpListener) serve(ctx context.Context) error {
 	tcaddr, err := net.ResolveTCPAddr(t.uri.Scheme, t.uri.Host)
 	if err != nil {
 		l.Infoln("Listen (BEP/tcp):", err)
@@ -76,7 +77,7 @@ func (t *tcpListener) serve(stop chan struct{}) error {
 		listener.SetDeadline(time.Now().Add(time.Second))
 		conn, err := listener.Accept()
 		select {
-		case <-stop:
+		case <-ctx.Done():
 			if err == nil {
 				conn.Close()
 			}
@@ -183,7 +184,7 @@ func (f *tcpListenerFactory) New(uri *url.URL, cfg config.Wrapper, tlsCfg *tls.C
 		natService: natService,
 		factory:    f,
 	}
-	l.ServiceWithError = util.AsServiceWithError(l.serve)
+	l.ServiceWithError = util.AsServiceWithError(l.serve, l.String())
 	return l
 }
 

lib/db/backend/backend.go

@@ -0,0 +1,170 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import (
+	"sync"
+)
+
+// The Reader interface specifies the read-only operations available on the
+// main database and on read-only transactions (snapshots). Note that when
+// called directly on the database handle these operations may take implicit
+// transactions and performance may suffer.
+type Reader interface {
+	Get(key []byte) ([]byte, error)
+	NewPrefixIterator(prefix []byte) (Iterator, error)
+	NewRangeIterator(first, last []byte) (Iterator, error)
+}
+
+// The Writer interface specifies the mutating operations available on the
+// main database and on writable transactions. Note that when called
+// directly on the database handle these operations may take implicit
+// transactions and performance may suffer.
+type Writer interface {
+	Put(key, val []byte) error
+	Delete(key []byte) error
+}
+
+// The ReadTransaction interface specifies the operations on read-only
+// transactions. Every ReadTransaction must be released when no longer
+// required.
+type ReadTransaction interface {
+	Reader
+	Release()
+}
+
+// The WriteTransaction interface specifies the operations on writable
+// transactions. Every WriteTransaction must be either committed or released
+// (i.e., discarded) when no longer required. No further operations must be
+// performed after release or commit (regardless of whether commit succeeded),
+// with one exception -- it's fine to release an already committed or released
+// transaction.
+//
+// A Checkpoint is a potential partial commit of the transaction so far, for
+// purposes of saving memory when transactions are in-RAM. Note that
+// transactions may be checkpointed *anyway* even if this is not called, due to
+// resource constraints, but this gives you a chance to decide when.
+type WriteTransaction interface {
+	ReadTransaction
+	Writer
+	Checkpoint() error
+	Commit() error
+}
+
+// The Iterator interface specifies the operations available on iterators
+// returned by NewPrefixIterator and NewRangeIterator. The iterator pattern
+// is to loop while Next returns true, then check Error after the loop. Next
+// will return false when iteration is complete (Error() == nil) or when
+// there is an error preventing iteration, which is then returned by
+// Error(). For example:
+//
+//     it, err := db.NewPrefixIterator(nil)
+//     if err != nil {
+//         // problem preventing iteration
+//     }
+//     defer it.Release()
+//     for it.Next() {
+//         // ...
+//     }
+//     if err := it.Error(); err != nil {
+//         // there was a database problem while iterating
+//     }
+//
+// An iterator must be Released when no longer required. The Error method
+// can be called either before or after Release with the same results. If an
+// iterator was created in a transaction (whether read-only or write) it
+// must be released before the transaction is released (or committed).
+type Iterator interface {
+	Next() bool
+	Key() []byte
+	Value() []byte
+	Error() error
+	Release()
+}
+
+// The Backend interface represents the main database handle. It supports
+// both read/write operations and opening read-only or writable
+// transactions. Depending on the actual implementation, individual
+// read/write operations may be implicitly wrapped in transactions, making
+// them perform quite badly when used repeatedly. For bulk operations,
+// consider always using a transaction of the appropriate type. The
+// transaction isolation level is "read committed" - there are no dirty
+// reads.
+type Backend interface {
+	Reader
+	Writer
+	NewReadTransaction() (ReadTransaction, error)
+	NewWriteTransaction() (WriteTransaction, error)
+	Close() error
+}
+
+type Tuning int
+
+const (
+	// N.b. these constants must match those in lib/config.Tuning!
+	TuningAuto Tuning = iota
+	TuningSmall
+	TuningLarge
+)
+
+func Open(path string, tuning Tuning) (Backend, error) {
+	return OpenLevelDB(path, tuning)
+}
+
+func OpenMemory() Backend {
+	return OpenLevelDBMemory()
+}
+
+type errClosed struct{}
+
+func (errClosed) Error() string { return "database is closed" }
+
+type errNotFound struct{}
+
+func (errNotFound) Error() string { return "key not found" }
+
+func IsClosed(err error) bool {
+	if _, ok := err.(errClosed); ok {
+		return true
+	}
+	if _, ok := err.(*errClosed); ok {
+		return true
+	}
+	return false
+}
+
+func IsNotFound(err error) bool {
+	if _, ok := err.(errNotFound); ok {
+		return true
+	}
+	if _, ok := err.(*errNotFound); ok {
+		return true
+	}
+	return false
+}
+
+// releaser manages counting on top of a waitgroup
+type releaser struct {
+	wg   *sync.WaitGroup
+	once *sync.Once
+}
+
+func newReleaser(wg *sync.WaitGroup) *releaser {
+	wg.Add(1)
+	return &releaser{
+		wg:   wg,
+		once: new(sync.Once),
+	}
+}
+
+func (r releaser) Release() {
+	// We use the Once because we may get called multiple times from
+	// Commit() and deferred Release().
+	r.once.Do(func() {
+		r.wg.Done()
+	})
+}

lib/db/backend/backend_test.go

@@ -0,0 +1,53 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import "testing"
+
+// testBackendBehavior is the generic test suite that must be fulfilled by
+// every backend implementation. It should be called by each implementation
+// as (part of) their test suite.
+func testBackendBehavior(t *testing.T, open func() Backend) {
+	t.Run("WriteIsolation", func(t *testing.T) { testWriteIsolation(t, open) })
+	t.Run("DeleteNonexisten", func(t *testing.T) { testDeleteNonexistent(t, open) })
+}
+
+func testWriteIsolation(t *testing.T, open func() Backend) {
+	// Values written during a transaction should not be read back, our
+	// updateGlobal depends on this.
+
+	db := open()
+	defer db.Close()
+
+	// Sanity check
+	_ = db.Put([]byte("a"), []byte("a"))
+	v, _ := db.Get([]byte("a"))
+	if string(v) != "a" {
+		t.Fatal("read back should work")
+	}
+
+	// Now in a transaction we should still see the old value
+	tx, _ := db.NewWriteTransaction()
+	defer tx.Release()
+	_ = tx.Put([]byte("a"), []byte("b"))
+	v, _ = tx.Get([]byte("a"))
+	if string(v) != "a" {
+		t.Fatal("read in transaction should read the old value")
+	}
+}
+
+func testDeleteNonexistent(t *testing.T, open func() Backend) {
+	// Deleting a non-existent key is not an error
+
+	db := open()
+	defer db.Close()
+
+	err := db.Delete([]byte("a"))
+	if err != nil {
+		t.Error(err)
+	}
+}

lib/db/backend/debug.go

@@ -0,0 +1,15 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import (
+	"github.com/syncthing/syncthing/lib/logger"
+)
+
+var (
+	l = logger.DefaultLogger.NewFacility("backend", "The database backend")
+)

lib/db/backend/leveldb_backend.go

@@ -0,0 +1,173 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package backend
+
+import (
+	"sync"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+const (
+	// Never flush transactions smaller than this, even on Checkpoint()
+	dbFlushBatchMin = 1 << MiB
+	// Once a transaction reaches this size, flush it unconditionally.
+	dbFlushBatchMax = 128 << MiB
+)
+
+// leveldbBackend implements Backend on top of a leveldb
+type leveldbBackend struct {
+	ldb     *leveldb.DB
+	closeWG sync.WaitGroup
+}
+
+func (b *leveldbBackend) NewReadTransaction() (ReadTransaction, error) {
+	return b.newSnapshot()
+}
+
+func (b *leveldbBackend) newSnapshot() (leveldbSnapshot, error) {
+	snap, err := b.ldb.GetSnapshot()
+	if err != nil {
+		return leveldbSnapshot{}, wrapLeveldbErr(err)
+	}
+	return leveldbSnapshot{
+		snap: snap,
+		rel:  newReleaser(&b.closeWG),
+	}, nil
+}
+
+func (b *leveldbBackend) NewWriteTransaction() (WriteTransaction, error) {
+	snap, err := b.newSnapshot()
+	if err != nil {
+		return nil, err // already wrapped
+	}
+	return &leveldbTransaction{
+		leveldbSnapshot: snap,
+		ldb:             b.ldb,
+		batch:           new(leveldb.Batch),
+		rel:             newReleaser(&b.closeWG),
+	}, nil
+}
+
+func (b *leveldbBackend) Close() error {
+	b.closeWG.Wait()
+	return wrapLeveldbErr(b.ldb.Close())
+}
+
+func (b *leveldbBackend) Get(key []byte) ([]byte, error) {
+	val, err := b.ldb.Get(key, nil)
+	return val, wrapLeveldbErr(err)
+}
+
+func (b *leveldbBackend) NewPrefixIterator(prefix []byte) (Iterator, error) {
+	return b.ldb.NewIterator(util.BytesPrefix(prefix), nil), nil
+}
+
+func (b *leveldbBackend) NewRangeIterator(first, last []byte) (Iterator, error) {
+	return b.ldb.NewIterator(&util.Range{Start: first, Limit: last}, nil), nil
+}
+
+func (b *leveldbBackend) Put(key, val []byte) error {
+	return wrapLeveldbErr(b.ldb.Put(key, val, nil))
+}
+
+func (b *leveldbBackend) Delete(key []byte) error {
+	return wrapLeveldbErr(b.ldb.Delete(key, nil))
+}
+
+// leveldbSnapshot implements backend.ReadTransaction
+type leveldbSnapshot struct {
+	snap *leveldb.Snapshot
+	rel  *releaser
+}
+
+func (l leveldbSnapshot) Get(key []byte) ([]byte, error) {
+	val, err := l.snap.Get(key, nil)
+	return val, wrapLeveldbErr(err)
+}
+
+func (l leveldbSnapshot) NewPrefixIterator(prefix []byte) (Iterator, error) {
+	return l.snap.NewIterator(util.BytesPrefix(prefix), nil), nil
+}
+
+func (l leveldbSnapshot) NewRangeIterator(first, last []byte) (Iterator, error) {
+	return l.snap.NewIterator(&util.Range{Start: first, Limit: last}, nil), nil
+}
+
+func (l leveldbSnapshot) Release() {
+	l.snap.Release()
+	l.rel.Release()
+}
+
+// leveldbTransaction implements backend.WriteTransaction using a batch (not
+// an actual leveldb transaction)
+type leveldbTransaction struct {
+	leveldbSnapshot
+	ldb   *leveldb.DB
+	batch *leveldb.Batch
+	rel   *releaser
+}
+
+func (t *leveldbTransaction) Delete(key []byte) error {
+	t.batch.Delete(key)
+	return t.checkFlush(dbFlushBatchMax)
+}
+
+func (t *leveldbTransaction) Put(key, val []byte) error {
+	t.batch.Put(key, val)
+	return t.checkFlush(dbFlushBatchMax)
+}
+
+func (t *leveldbTransaction) Checkpoint() error {
+	return t.checkFlush(dbFlushBatchMin)
+}
+
+func (t *leveldbTransaction) Commit() error {
+	err := wrapLeveldbErr(t.flush())
+	t.leveldbSnapshot.Release()
+	t.rel.Release()
+	return err
+}
+
+func (t *leveldbTransaction) Release() {
+	t.leveldbSnapshot.Release()
+	t.rel.Release()
+}
+
+// checkFlush flushes and resets the batch if its size exceeds the given size.
+func (t *leveldbTransaction) checkFlush(size int) error {
+	if len(t.batch.Dump()) < size {
+		return nil
+	}
+	return t.flush()
+}
+
+func (t *leveldbTransaction) flush() error {
+	if t.batch.Len() == 0 {
+		return nil
+	}
+	if err := t.ldb.Write(t.batch, nil); err != nil {
+		return wrapLeveldbErr(err)
+	}
+	t.batch.Reset()
+	return nil
+}
+
+// wrapLeveldbErr wraps errors so that the backend package can recognize them
+func wrapLeveldbErr(err error) error {
+	if err == nil {
+		return nil
+	}
+	if err == leveldb.ErrClosed {
+		return errClosed{}
+	}
+	if err == leveldb.ErrNotFound {
+		return errNotFound{}
+	}
+	return err
+}