vendor: Update github.com/syncthing/notify (fixes #4885) (#4894)

With slight modifications

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+/AUTHORS    @calmh
+/*.md       @calmh

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,34 @@
+### DO NOT REPORT SECURITY ISSUES IN THIS ISSUE TRACKER
+
+Instead, contact security@syncthing.net directly - see
+https://syncthing.net/security.html for more information.
+
+### DO NOT POST SUPPORT REQUESTS OR GENERAL QUESTIONS IN THIS ISSUE TRACKER
+
+Please use the forum at https://forum.syncthing.net/ where a large number of
+helpful people hang out. This issue tracker is for reporting bugs or feature
+requests directly to the developers. Worst case you might get a short
+"that's a bug, please report it on GitHub" response on the forum, in which
+case we thank you for your patience and following our advice. :)
+
+### Please do post actual bug reports and feature requests.
+
+If your issue is a bug report, replace this boilerplate with a description
+of the problem, being sure to include at least:
+
+ - what happened,
+ - what you expected to happen instead, and
+ - any steps to reproduce the problem.
+
+Also fill out the version information below and add log output or
+screenshots as appropriate.
+
+If your issue is a feature request, simply replace this template text in
+its entirety.
+
+### Version Information
+
+Syncthing Version: v0.x.y
+OS Version: Windows 7 / Ubuntu 14.04 / ...
+Browser Version: (if applicable, for GUI issues)
+

.mailmap

@@ -1 +0,0 @@
-NICKS

AUTHORS

@@ -3,7 +3,9 @@
 #
 #    Name Name Name (nickname) <email1@example.com> <email2@example.com>
 #
-# The NICKS list is auto generated from this file.
+# After changing this list, run "go run script/authors.go" to sort and update
+# the GUI HTML.
+#
 
 Aaron Bieber (qbit) <qbit@deftly.net>
 Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com>
@@ -21,6 +23,7 @@ Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com>
 Bart De Vries (mogwa1) <devriesb@gmail.com>
 Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
 Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
+Ben Shepherd (benshep) <bjashepherd@gmail.com>
 Ben Sidhom (bsidhom) <bsidhom@gmail.com>
 Benny Ng (tpng) <benny.tpng@gmail.com>
 Brandon Philips (philips) <brandon@ifup.org>
@@ -36,9 +39,11 @@ Colin Kennedy (moshen) <moshen.colin@gmail.com>
 Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
 Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
 Daniel Martí (mvdan) <mvdan@mvdan.cc>
+Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
 David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
 Denis A. (dva) <denisva@gmail.com>
 Dennis Wilson (snnd) <dw@risu.io>
+Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
 Dominik Heidler (asdil12) <dominik@heidler.eu>
 Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
 Emil Hessman (ceh) <emil@hessman.se>
@@ -49,6 +54,7 @@ Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
 Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
 Frank Isemann (fti7) <frank@isemann.name>
 Gilli Sigurdsson (gillisig) <gilli@vx.is>
+Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
 Heiko Zuerker (Smiley73) <heiko@zuerker.org>
 Jaakko Hannikainen (jgke) <jgke@jgke.fi>
 Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
@@ -56,18 +62,22 @@ Jake Peterson (acogdev) <jake@acogdev.com>
 Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
 James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
 Jaroslav Malec (dzarda) <dzardacz@gmail.com>
+Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
 Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
 Jochen Voss (seehuhn) <voss@seehuhn.de>
 Johan Vromans (sciurius) <jvromans@squirrel.nl>
+John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
+Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
 Karol Różycki (krozycki) <rozycki.karol@gmail.com>
 Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
 Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
 Kevin Allen (ironmig) <kma1660@gmail.com>
 Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
-Kurt Fitzner (Kudalufi) <kurt@va1der.ca>
+Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
 Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
 Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
 Leo Arias (elopio) <yo@elopio.net>
+Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
 Lode Hoste (Zillode) <zillode@zillode.be>
 Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
 Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
@@ -82,21 +92,30 @@ Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
 Michael Ploujnikov (plouj) <ploujj@gmail.com>
 Michael Tilli (pyfisch) <pyfisch@gmail.com>
 Nate Morrison (nrm21) <natemorrison@gmail.com>
+Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
+Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
 Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
+Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
 Peter Hoeg (peterhoeg) <peter@speartail.com>
 Philippe Schommers (filoozoom) <philippe@schommers.be>
 Phill Luby (pluby) <phill.luby@newredo.com>
 Piotr Bejda (piobpl) <piotrb10@gmail.com>
+Robert Carosi (nov1n) <robert@carosi.nl>
 Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
+Ross Smith II (rasa) <ross@smithii.com>
 Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
+Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
 Scott Klupfel (kluppy) <kluppy@going2blue.com>
 Sergey Mishin (ralder) <ralder@yandex.ru>
 Simon Frei (imsodin) <freisim93@gmail.com>
 Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
 Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
+Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
+Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
 Tim Abell (timabell) <tim@timwise.co.uk>
 Tim Howes (timhowes) <timhowes@berkeley.edu>
 Tobias Nygren (tnn2) <tnn@nygren.pp.se>
+Tobias Tom (tobiastom) <t.tom@succont.de>
 Tomas Cerveny (kozec) <kozec@kozec.com>
 Tully Robinson (tojrobinson) <tully@tojr.org>
 Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
@@ -105,6 +124,6 @@ Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
 Victor Buinsky (buinsky) <vix_booja@tut.by>
 Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
 William A. Kennington III (wkennington) <william@wkennington.com>
-Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de>
+Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
 Xavier O. (damajor) <damajor@gmail.com>
 Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>

Dockerfile

@@ -0,0 +1,29 @@
+FROM golang:1.10 AS builder
+
+WORKDIR /go/src/github.com/syncthing/syncthing
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f syncthing && go run build.go build syncthing
+
+FROM alpine
+
+RUN apk add --no-cache ca-certificates
+
+COPY --from=builder /go/src/github.com/syncthing/syncthing/syncthing /bin/syncthing
+
+RUN echo 'syncthing:x:1000:1000::/var/syncthing:/sbin/nologin' >> /etc/passwd \
+    && echo 'syncthing:!::0:::::' >> /etc/shadow \
+    && mkdir /var/syncthing \
+    && chown syncthing /var/syncthing
+
+USER syncthing
+ENV STNOUPGRADE=1
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 8384 || exit 1
+
+ENTRYPOINT ["/bin/syncthing", "-home", "/var/syncthing/config", "-gui-address", "0.0.0.0:8384"]
+

GOALS.md

@@ -1,6 +1,6 @@
 # The Syncthing Goals
 
-Syncthing is a **continous file synchronization program**. It synchronizes
+Syncthing is a **continuous file synchronization program**. It synchronizes
 files between two or more computers. We strive to fulfill the goals below.
 The goals are listed in order of importance, the most important one being
 the first.
@@ -36,7 +36,7 @@ or modification by unauthorized parties.
 
 Syncthing should be approachable, understandable and inclusive.
 
-> Complex concepts and maths form the base of Synchting's functionality.
+> Complex concepts and maths form the base of Syncthing's functionality.
 > This should nonetheless be abstracted or hidden to a degree where
 > Syncthing is usable by the general public.
 
@@ -80,4 +80,4 @@ conflict with the stated goals above.
 > providing entertainment value for the maintainers are also things that
 > matter. It is understood that there are aspects of Syncthing that are
 > suboptimal or even in opposition with the goals above. However, we
-> continously strive to align Syncthing more and more with these goals.
+> continuously strive to align Syncthing more and more with these goals.

ISSUE_TEMPLATE.md

@@ -1,28 +0,0 @@
-Do not report security issues in this bug tracker. Instead, contact
-security@syncthing.net directly - see https://syncthing.net/security.html
-for more information.
-
-If your issue is a support request ("How do I get my devices to connect?"
-or similar), please use the support forum at https://forum.syncthing.net/
-where a large number of helpful people hang out. This issue tracker is for
-reporting bugs or feature requests directly to the developers.
-
-If your issue is a bug report, replace this boilerplate with a description
-of the problem, being sure to include at least:
-
- - what happened,
- - what you expected to happen instead, and
- - any steps to reproduce the problem.
-
-Also fill out the version information below and add log output or
-screenshots as appropriate.
-
-If your issue is a feature request, simply replace this template text in
-its entirety.
-
-### Version Information
-
-Syncthing Version: v0.x.y
-OS Version: Windows 7 / Ubuntu 14.04 / ...
-Browser Version: (if applicable, for GUI issues)
-

LICENSE

@@ -357,7 +357,7 @@ Exhibit A - Source Code Form License Notice
 
   This Source Code Form is subject to the terms of the Mozilla Public
   License, v. 2.0. If a copy of the MPL was not distributed with this
-  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+  file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 If it is not possible or desirable to put the notice in a particular
 file, then You may include the notice in a location (such as a LICENSE

NICKS

@@ -1,132 +0,0 @@
-# This file maps email addresses used in commits to nicks used the changelog.
-# It is auto generated from the AUTHORS file by script/authors.go.
-
-0x010C <antoine.lamielle@0x010c.fr>
-0x010C <gh@0x010c.fr>
-acogdev <jake@acogdev.com>
-adelq <aqalieh95@gmail.com>
-adelq <adelq@users.noreply.github.com>
-alessandro.g89 <alessandro.g89@gmail.com>
-alex2108 <register-github@alex-graf.de>
-andersonvom <andersonvom@gmail.com>
-andrew-d <andrew@du.nham.ca>
-asdil12 <dominik@heidler.eu>
-AudriusButkevicius <audrius.butkevicius@gmail.com>
-aviau <alexandre@alexandreviau.net>
-aviau <aviau@debian.org>
-bencurthoys <ben@bencurthoys.com>
-bigbear2nd <bigbear2nd@gmail.com>
-brbecker <brbecker@gmail.com>
-brendanlong <self@brendanlong.com>
-brgmnn <dan.arne.bergmann@gmail.com>
-brgmnn <brgmnn@users.noreply.github.com>
-bsidhom <bsidhom@gmail.com>
-buinsky <vix_booja@tut.by>
-burkemw3 <mburke@amplify.com>
-burkemw3 <burkemw3@gmail.com>
-calmh <jakob@nym.se>
-calmh <jakob@kastelo.net>
-canton7 <antony.male@gmail.com>
-Cathryne <cathryne.linenweaver@gmail.com>
-Cathryne <Cathryne@users.noreply.github.com>
-cdata <chris@scriptolo.gy>
-cdhowie <me@chrishowie.com>
-ceh <emil@hessman.se>
-cqcallaw <enlightened.despot@gmail.com>
-damajor <damajor@gmail.com>
-dinosore <dinosore@dbrsoftware.co.uk>
-dva <denisva@gmail.com>
-dzarda <dzardacz@gmail.com>
-eipiminus1 <eipiminusone+github@gmail.com>
-eipiminus1 <eipiminus1@users.noreply.github.com>
-elopio <yo@elopio.net>
-facastagnini <federico.castagnini@gmail.com>
-filoozoom <philippe@schommers.be>
-frioux <frew@afoolishmanifesto.com>
-frioux <frioux@gmail.com>
-fti7 <frank@isemann.name>
-gillisig <gilli@vx.is>
-hadogenes <szafar@linux.pl>
-imsodin <freisim93@gmail.com>
-ironmig <kma1660@gmail.com>
-jarlebring <jarlebring@gmail.com>
-jedie <github.com@jensdiemer.de>
-jedie <git@jensdiemer.de>
-jgke <jgke@jgke.fi>
-jpjp <jamespatterson@operamail.com>
-jpjp <jpjp@users.noreply.github.com>
-kamadak <kamada@nanohz.org>
-KayoticSully <kayoticsully@gmail.com>
-kc1212 <kc04bc@gmx.com>
-kc1212 <kc1212@users.noreply.github.com>
-kilburn <kilburn@la3.org>
-kluppy <kluppy@going2blue.com>
-kozec <kozec@kozec.com>
-kralo <max.schulze@online.de>
-kralo <kralo@users.noreply.github.com>
-krozycki <rozycki.karol@gmail.com>
-Kudalufi <kurt@va1der.ca>
-kwhite17 <kevinwhite1710@gmail.com>
-letiemble <laurent.etiemble@gmail.com>
-letiemble <laurent.etiemble@monobjc.net>
-lkwg82 <lkwg82@gmx.de>
-LordLandon <lordlandon@gmail.com>
-majedev <majed.alhajry@gmail.com>
-marcindziadus <dziadus.marcin@gmail.com>
-marclaporte <marc@marclaporte.com>
-marclaporte <marc@laporte.name>
-mateon1 <matin1111@wp.pl>
-mogwa1 <devriesb@gmail.com>
-moshen <moshen.colin@gmail.com>
-Moter8 <moter8@gmail.com>
-mpx <mark@kyne.com.au>
-mvdan <mvdan@mvdan.cc>
-norgeous <daniel@harte.me>
-norgeous <daniel@danielharte.co.uk>
-norgeous <norgeous@users.noreply.github.com>
-nrm21 <natemorrison@gmail.com>
-Nutomic <me@nutomic.com>
-pascalj <github@pascalj.com>
-pascalj <mail@pascal-jungblut.com>
-peterhoeg <peter@speartail.com>
-philips <brandon@ifup.org>
-piobpl <piotrb10@gmail.com>
-plouj <ploujj@gmail.com>
-pluby <phill.luby@newredo.com>
-ProactiveServices <aD@simplypeachy.co.uk>
-ProactiveServices <simplypeachy@users.noreply.github.com>
-ProactiveServices <ProactiveServices@users.noreply.github.com>
-pyfisch <pyfisch@gmail.com>
-qbit <qbit@deftly.net>
-ralder <ralder@yandex.ru>
-Rewt0r <rewt0r@gmx.com>
-Rewt0r <Rewt0r@users.noreply.github.com>
-rumpelsepp <stefan@sevenbyte.org>
-rumpelsepp <rumpelsepp@sevenbyte.org>
-scienmind <scintertech@cryptolab.net>
-sciurius <jvromans@squirrel.nl>
-seehuhn <voss@seehuhn.de>
-Smiley73 <heiko@zuerker.org>
-snnd <dw@risu.io>
-Stefan-Code <stefan.github@gmail.com>
-Stefan-Code <Stefan.github@gmail.com>
-timabell <tim@timwise.co.uk>
-timhowes <timhowes@berkeley.edu>
-tnn2 <tnn@nygren.pp.se>
-tojrobinson <tully@tojr.org>
-tpng <benny.tpng@gmail.com>
-tylerbrazier <tyler@tylerbrazier.com>
-Unrud <unrud@openaliasbox.org>
-Unrud <Unrud@users.noreply.github.com>
-uok <ueomkail@gmail.com>
-uok <uok@users.noreply.github.com>
-veeti <veeti.paananen@rojekti.fi>
-Vilbrekin <vilbrekin@gmail.com>
-wkennington <william@wkennington.com>
-WSGCSysadmin <e.meitner@willystreet.coop>
-wweich <wweich@users.noreply.github.com>
-wweich <wweich@gmx.de>
-xduugu <cedric@gmx.ca>
-zaynetro <romanznet@gmail.com>
-Zillode <zillode@zillode.be>
-zukoo <fxgsell@gmail.com>

README-Docker.md

@@ -0,0 +1,43 @@
+# Docker Container for Syncthing
+
+Use the Dockerfile in this repo, or pull the `syncthing/syncthing` image
+from Docker Hub. Use volumes to have the synchronized files available on the
+host.
+
+The exposed volumes are by default:
+
+    /var/syncthing/config   - the configuration and index directory into the Container
+    /var/syncthing          - the default sync folder into the Container
+
+You can add more folders and map them as you prefer.
+
+Note that Syncthing runs as UID 1000 in the container. This UID must have
+permission to read and modify the files in the containers.
+
+Example usage:
+
+```
+$ docker pull syncthing/syncthing
+$ docker run -p 8384:8384 -p 22000:22000 \
+    -v /wherever/st-cfg:/var/syncthing/config \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+Note that local device discovery will not work with the above command resulting
+in poor local transfer rates if local device addresses are not manually
+configured.
+
+To allow local discovery, the docker host network can be used instead:
+
+```
+$ docker pull syncthing/syncthing
+$ docker run --network=host \
+    -v /wherever/st-cfg:/var/syncthing/config \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+Be aware that syncthing alone is now in control of what interfaces and ports it
+listens on. You can edit the syncthing configuration to change the defaults if
+there are conflicts.

README.md

@@ -1,13 +1,18 @@
-# Syncthing
+[![Syncthing][14]][15]
 
-[![Latest Build (Official)](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square&label=unix%20build)](http://build.syncthing.net/job/syncthing/lastBuild/)
-[![API Documentation](https://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/syncthing)
+---
+
+[![Latest Downloads](https://img.shields.io/badge/latest-downloads-brightgreen.svg?style=flat-square)](https://build.syncthing.net/latest/)
+[![Latest Linux & Cross Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildLinuxCross.svg?style=flat-square&label=linux+%26+cross+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildLinuxCross&guest=1)
+[![Latest Windows Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildWindows.svg?style=flat-square&label=windows+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildWindows&guest=1)
+[![Latest Mac Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildMac.svg?style=flat-square&label=mac+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildMac&guest=1)
 [![MPLv2 License](https://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/88/badge)](https://bestpractices.coreinfrastructure.org/projects/88)
+[![Go Report Card](https://goreportcard.com/badge/github.com/syncthing/syncthing)](https://goreportcard.com/report/github.com/syncthing/syncthing)
 
 ## Goals
 
-Syncthing is a **continous file synchronization program**. It synchronizes
+Syncthing is a **continuous file synchronization program**. It synchronizes
 files between two or more computers. We strive to fulfill the goals below.
 The goals are listed in order of importance, the most important one being
 the first. This is the summary version of the goal list - for more
@@ -15,36 +20,36 @@ commentary, see the full [Goals document][13].
 
 Syncthing should be:
 
-1. Safe From Data Loss
+1. **Safe From Data Loss**
 
    Protecting the user's data is paramount. We take every reasonable
    precaution to avoid corrupting the user's files.
 
-2. Secure Against Attackers
+2. **Secure Against Attackers**
 
    Again, protecting the user's data is paramount. Regardless of our other
    goals we must never allow the user's data to be susceptible to
    eavesdropping or modification by unauthorized parties.
 
-3. Easy to Use
+3. **Easy to Use**
 
    Syncthing should be approachable, understandable and inclusive.
 
-4. Automatic
+4. **Automatic**
 
    User interaction should be required only when absolutely necessary.
 
-5. Universally Available
+5. **Universally Available**
 
    Syncthing should run on every common computer. We are mindful that the
    latest technology is not always available to any given individual.
 
-6. For Individuals
+6. **For Individuals**
 
    Syncthing is primarily about empowering the individual user with safe,
    secure and easy to use file synchronization.
 
-7. Everything Else
+7. **Everything Else**
 
    There are many things we care about that don't make it on to the list. It
    is fine to optimize for these values, as long as they are not in conflict
@@ -82,8 +87,8 @@ D26E6ED000654A3E, available from https://syncthing.net/security.html and
 most key servers.
 
 There is also a built in automatic upgrade mechanism (disabled in some
-distribution channels) which uses a compiled in ECDSA signature. Mac OS
-X binaries are also properly code signed.
+distribution channels) which uses a compiled in ECDSA signature. macOS
+binaries are also properly code signed.
 
 ## Documentation
 
@@ -94,7 +99,7 @@ All code is licensed under the [MPLv2 License][7].
 [1]: https://docs.syncthing.net/specs/bep-v1.html
 [2]: https://docs.syncthing.net/intro/getting-started.html
 [3]: https://github.com/syncthing/syncthing/blob/master/etc
-[4]: http://www.freenode.net/irc_servers.shtml
+[4]: https://www.freenode.net/
 [5]: https://docs.syncthing.net/dev/building.html
 [6]: https://docs.syncthing.net/
 [7]: https://github.com/syncthing/syncthing/blob/master/LICENSE
@@ -104,3 +109,6 @@ All code is licensed under the [MPLv2 License][7].
 [11]: https://docs.syncthing.net/users/contrib.html#gui-wrappers
 [12]: https://www.bountysource.com/teams/syncthing/issues
 [13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
+[14]: assets/logo-text-128.png
+[15]: https://syncthing.net/
+

build.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 // +build ignore
 
@@ -12,7 +12,9 @@ import (
 	"archive/tar"
 	"archive/zip"
 	"bytes"
+	"compress/flate"
 	"compress/gzip"
+	"crypto/sha256"
 	"errors"
 	"flag"
 	"fmt"
@@ -32,18 +34,27 @@ import (
 )
 
 var (
-	versionRe = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
-	goarch    string
-	goos      string
-	noupgrade bool
-	version   string
-	goVersion float64
-	race      bool
-	debug     = os.Getenv("BUILDDEBUG") != ""
+	versionRe     = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
+	goarch        string
+	goos          string
+	noupgrade     bool
+	version       string
+	goVersion     float64
+	race          bool
+	debug         = os.Getenv("BUILDDEBUG") != ""
+	noBuildGopath bool
+	extraTags     string
+	installSuffix string
+	pkgdir        string
+	debugBinary   bool
 )
 
 type target struct {
 	name              string
+	debname           string
+	debdeps           []string
+	debpost           string
+	description       string
 	buildPkg          string
 	binaryName        string
 	archiveFiles      []archiveFile
@@ -61,14 +72,18 @@ var targets = map[string]target{
 	"all": {
 		// Only valid for the "build" and "install" commands as it lacks all
 		// the archive creation stuff.
-		buildPkg: "./cmd/...",
+		buildPkg: "github.com/syncthing/syncthing/cmd/...",
 		tags:     []string{"purego"},
 	},
 	"syncthing": {
 		// The default target for "build", "install", "tar", "zip", "deb", etc.
-		name:       "syncthing",
-		buildPkg:   "./cmd/syncthing",
-		binaryName: "syncthing", // .exe will be added automatically for Windows builds
+		name:        "syncthing",
+		debname:     "syncthing",
+		debdeps:     []string{"libc6", "procps"},
+		debpost:     "script/post-upgrade",
+		description: "Open Source Continuous File Synchronization",
+		buildPkg:    "github.com/syncthing/syncthing/cmd/syncthing",
+		binaryName:  "syncthing", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
 			{src: "README.md", dst: "README.txt", perm: 0644},
@@ -98,28 +113,34 @@ var targets = map[string]target{
 		},
 	},
 	"stdiscosrv": {
-		name:       "stdiscosrv",
-		buildPkg:   "./cmd/stdiscosrv",
-		binaryName: "stdiscosrv", // .exe will be added automatically for Windows builds
+		name:        "stdiscosrv",
+		debname:     "syncthing-discosrv",
+		debdeps:     []string{"libc6"},
+		description: "Syncthing Discovery Server",
+		buildPkg:    "github.com/syncthing/syncthing/cmd/stdiscosrv",
+		binaryName:  "stdiscosrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
 			{src: "cmd/stdiscosrv/README.md", dst: "README.txt", perm: 0644},
-			{src: "cmd/stdiscosrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
-			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/stdiscosrv/README.txt", perm: 0644},
-			{src: "cmd/stdiscosrv/LICENSE", dst: "deb/usr/share/doc/stdiscosrv/LICENSE.txt", perm: 0644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/stdiscosrv/AUTHORS.txt", perm: 0644},
+			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0644},
+			{src: "cmd/stdiscosrv/LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0644},
 			{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0644},
 		},
 		tags: []string{"purego"},
 	},
 	"strelaysrv": {
-		name:       "strelaysrv",
-		buildPkg:   "./cmd/strelaysrv",
-		binaryName: "strelaysrv", // .exe will be added automatically for Windows builds
+		name:        "strelaysrv",
+		debname:     "syncthing-relaysrv",
+		debdeps:     []string{"libc6"},
+		description: "Syncthing Relay Server",
+		buildPkg:    "github.com/syncthing/syncthing/cmd/strelaysrv",
+		binaryName:  "strelaysrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
 			{src: "cmd/strelaysrv/README.md", dst: "README.txt", perm: 0644},
@@ -128,16 +149,19 @@ var targets = map[string]target{
 		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
-			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/strelaysrv/README.txt", perm: 0644},
-			{src: "cmd/strelaysrv/LICENSE", dst: "deb/usr/share/doc/strelaysrv/LICENSE.txt", perm: 0644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/strelaysrv/AUTHORS.txt", perm: 0644},
+			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0644},
+			{src: "cmd/strelaysrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0644},
 			{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0644},
 		},
 	},
 	"strelaypoolsrv": {
-		name:       "strelaypoolsrv",
-		buildPkg:   "./cmd/strelaypoolsrv",
-		binaryName: "strelaypoolsrv", // .exe will be added automatically for Windows builds
+		name:        "strelaypoolsrv",
+		debname:     "syncthing-relaypoolsrv",
+		debdeps:     []string{"libc6"},
+		description: "Syncthing Relay Pool Server",
+		buildPkg:    "github.com/syncthing/syncthing/cmd/strelaypoolsrv",
+		binaryName:  "strelaypoolsrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
 			{src: "cmd/strelaypoolsrv/README.md", dst: "README.txt", perm: 0644},
@@ -146,47 +170,13 @@ var targets = map[string]target{
 		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
-			{src: "cmd/strelaypoolsrv/README.md", dst: "deb/usr/share/doc/relaysrv/README.txt", perm: 0644},
-			{src: "cmd/strelaypoolsrv/LICENSE", dst: "deb/usr/share/doc/relaysrv/LICENSE.txt", perm: 0644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/relaysrv/AUTHORS.txt", perm: 0644},
+			{src: "cmd/strelaypoolsrv/README.md", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/README.txt", perm: 0644},
+			{src: "cmd/strelaypoolsrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/AUTHORS.txt", perm: 0644},
 		},
 	},
 }
 
-var (
-	// fast linters complete in a fraction of a second and might as well be
-	// run always as part of the build
-	fastLinters = []string{
-		"deadcode",
-		"golint",
-		"ineffassign",
-		"vet",
-	}
-
-	// slow linters take several seconds and are run only as part of the
-	// "metalint" command.
-	slowLinters = []string{
-		"gosimple",
-		"staticcheck",
-		"structcheck",
-		"unused",
-		"varcheck",
-	}
-
-	// Which parts of the tree to lint
-	lintDirs = []string{".", "./lib/...", "./cmd/..."}
-
-	// Messages to ignore
-	lintExcludes = []string{
-		".pb.go",
-		"should have comment",
-		"protocol.Vector composite literal uses unkeyed fields",
-		"cli.Requires composite literal uses unkeyed fields",
-		"Use DialContext instead",   // Go 1.7
-		"os.SEEK_SET is deprecated", // Go 1.7
-	}
-)
-
 func init() {
 	// The "syncthing" target includes a few more files found in the "etc"
 	// and "extra" dirs.
@@ -203,12 +193,11 @@ func init() {
 	targets["syncthing"] = syncthingPkg
 }
 
-const minGoVersion = 1.5
-
 func main() {
-	log.SetOutput(os.Stdout)
 	log.SetFlags(0)
 
+	parseFlags()
+
 	if debug {
 		t0 := time.Now()
 		defer func() {
@@ -216,21 +205,40 @@ func main() {
 		}()
 	}
 
-	if os.Getenv("GOPATH") == "" {
-		setGoPath()
+	if gopath := gopath(); gopath == "" {
+		gopath, err := temporaryBuildDir()
+		if err != nil {
+			log.Fatal(err)
+		}
+		if !noBuildGopath {
+			lazyRebuildAssets()
+			if err := buildGOPATH(gopath); err != nil {
+				log.Fatal(err)
+			}
+		}
+		os.Setenv("GOPATH", gopath)
+		log.Println("GOPATH is", gopath)
+	} else {
+		inside := false
+		wd, _ := os.Getwd()
+		wd, _ = filepath.EvalSymlinks(wd)
+		for _, p := range filepath.SplitList(gopath) {
+			p, _ = filepath.EvalSymlinks(p)
+			if filepath.Join(p, "src/github.com/syncthing/syncthing") == wd {
+				inside = true
+				break
+			}
+		}
+		if !inside {
+			fmt.Println("You seem to have GOPATH set but the Syncthing source not placed correctly within it, which may cause problems.")
+		}
 	}
 
-	// We use Go 1.5+ vendoring.
-	os.Setenv("GO15VENDOREXPERIMENT", "1")
-
 	// Set path to $GOPATH/bin:$PATH so that we can for sure find tools we
 	// might have installed during "build.go setup".
 	os.Setenv("PATH", fmt.Sprintf("%s%cbin%c%s", os.Getenv("GOPATH"), os.PathSeparator, os.PathListSeparator, os.Getenv("PATH")))
 
-	parseFlags()
-
 	checkArchitecture()
-	goVersion, _ = checkRequiredGoVersion()
 
 	// Invoking build.go with no parameters at all builds everything (incrementally),
 	// which is what you want for maximum error checking during development.
@@ -255,7 +263,7 @@ func main() {
 
 func checkArchitecture() {
 	switch goarch {
-	case "386", "amd64", "arm", "arm64", "ppc64", "ppc64le":
+	case "386", "amd64", "arm", "arm64", "ppc64", "ppc64le", "mips", "mipsle":
 		break
 	default:
 		log.Printf("Unknown goarch %q; proceed with caution!", goarch)
@@ -272,22 +280,23 @@ func runCommand(cmd string, target target) {
 		if noupgrade {
 			tags = []string{"noupgrade"}
 		}
+		tags = append(tags, strings.Fields(extraTags)...)
 		install(target, tags)
-		metalint(fastLinters, lintDirs)
+		metalintShort()
 
 	case "build":
 		var tags []string
 		if noupgrade {
 			tags = []string{"noupgrade"}
 		}
+		tags = append(tags, strings.Fields(extraTags)...)
 		build(target, tags)
-		metalint(fastLinters, lintDirs)
 
 	case "test":
-		test("./lib/...", "./cmd/...")
+		test("github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
 
 	case "bench":
-		bench("./lib/...", "./cmd/...")
+		bench("github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
 
 	case "assets":
 		rebuildAssets()
@@ -317,67 +326,43 @@ func runCommand(cmd string, target target) {
 		clean()
 
 	case "vet":
-		metalint(fastLinters, lintDirs)
+		metalintShort()
 
 	case "lint":
-		metalint(fastLinters, lintDirs)
+		metalintShort()
 
 	case "metalint":
-		metalint(fastLinters, lintDirs)
-		metalint(slowLinters, lintDirs)
+		metalint()
 
 	case "version":
 		fmt.Println(getVersion())
 
+	case "gopath":
+		gopath, err := temporaryBuildDir()
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Println(gopath)
+
 	default:
 		log.Fatalf("Unknown command %q", cmd)
 	}
 }
 
-// setGoPath sets GOPATH correctly with the assumption that we are
-// in $GOPATH/src/github.com/syncthing/syncthing.
-func setGoPath() {
-	cwd, err := os.Getwd()
-	if err != nil {
-		log.Fatal(err)
-	}
-	gopath := filepath.Clean(filepath.Join(cwd, "../../../../"))
-	log.Println("GOPATH is", gopath)
-	os.Setenv("GOPATH", gopath)
-}
-
 func parseFlags() {
 	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
 	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
 	flag.BoolVar(&noupgrade, "no-upgrade", noupgrade, "Disable upgrade functionality")
 	flag.StringVar(&version, "version", getVersion(), "Set compiled in version string")
 	flag.BoolVar(&race, "race", race, "Use race detector")
+	flag.BoolVar(&noBuildGopath, "no-build-gopath", noBuildGopath, "Don't build GOPATH, assume it's OK")
+	flag.StringVar(&extraTags, "tags", extraTags, "Extra tags, space separated")
+	flag.StringVar(&installSuffix, "installsuffix", installSuffix, "Install suffix, optional")
+	flag.StringVar(&pkgdir, "pkgdir", "", "Set -pkgdir parameter for `go build`")
+	flag.BoolVar(&debugBinary, "debug-binary", debugBinary, "Create unoptimized binary to use with delve, set -gcflags='-N -l' and omit -ldflags")
 	flag.Parse()
 }
 
-func checkRequiredGoVersion() (float64, bool) {
-	re := regexp.MustCompile(`go(\d+\.\d+)`)
-	ver := runtime.Version()
-	if m := re.FindStringSubmatch(ver); len(m) == 2 {
-		vs := string(m[1])
-		// This is a standard go build. Verify that it's new enough.
-		f, err := strconv.ParseFloat(vs, 64)
-		if err != nil {
-			log.Printf("*** Couldn't parse Go version out of %q.\n*** This isn't known to work, proceed on your own risk.", vs)
-			return 0, false
-		}
-		if f < 1.5 {
-			log.Printf("*** Go version %.01f doesn't support the vendoring mechanism.\n*** Ensure correct dependencies in your $GOPATH.", f)
-		} else if f < minGoVersion {
-			log.Fatalf("*** Go version %.01f is less than required %.01f.\n*** This is known not to work, not proceeding.", f, minGoVersion)
-		}
-		return f, true
-	}
-
-	log.Printf("*** Unknown Go version %q.\n*** This isn't known to work, proceed on your own risk.", ver)
-	return 0, false
-}
-
 func setup() {
 	packages := []string{
 		"github.com/alecthomas/gometalinter",
@@ -392,16 +377,17 @@ func setup() {
 		"github.com/tsenart/deadcode",
 		"golang.org/x/net/html",
 		"golang.org/x/tools/cmd/cover",
-		"honnef.co/go/simple/cmd/gosimple",
-		"honnef.co/go/staticcheck/cmd/staticcheck",
-		"honnef.co/go/unused/cmd/unused",
+		"honnef.co/go/tools/cmd/gosimple",
+		"honnef.co/go/tools/cmd/staticcheck",
+		"honnef.co/go/tools/cmd/unused",
+		"github.com/josephspurrier/goversioninfo",
 	}
 	for _, pkg := range packages {
 		fmt.Println(pkg)
 		runPrint("go", "get", "-u", pkg)
 	}
 
-	runPrint("go", "install", "-v", "./vendor/github.com/gogo/protobuf/protoc-gen-gogofast")
+	runPrint("go", "install", "-v", "github.com/syncthing/syncthing/vendor/github.com/gogo/protobuf/protoc-gen-gogofast")
 }
 
 func test(pkgs ...string) {
@@ -436,17 +422,23 @@ func install(target target, tags []string) {
 		log.Fatal(err)
 	}
 	os.Setenv("GOBIN", filepath.Join(cwd, "bin"))
-	args := []string{"install", "-v", "-ldflags", ldflags()}
-	if len(tags) > 0 {
-		args = append(args, "-tags", strings.Join(tags, " "))
-	}
-	if race {
-		args = append(args, "-race")
-	}
-	args = append(args, target.buildPkg)
+
+	args := []string{"install", "-v"}
+	args = appendParameters(args, tags, target)
 
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
+
+	// On Windows generate a special file which the Go compiler will automatically use when generating Windows binaries
+	// to set things like the file icon, version, etc.
+	if goos == "windows" {
+		sysoPath, err := shouldBuildSyso(cwd)
+		if err != nil {
+			log.Printf("Warning: Windows binaries will not have file information encoded: %v", err)
+		}
+		defer shouldCleanupSyso(sysoPath)
+	}
+
 	runPrint("go", args...)
 }
 
@@ -455,21 +447,44 @@ func build(target target, tags []string) {
 
 	tags = append(target.tags, tags...)
 
-	rmr(target.binaryName)
-	args := []string{"build", "-i", "-v", "-ldflags", ldflags()}
-	if len(tags) > 0 {
-		args = append(args, "-tags", strings.Join(tags, " "))
-	}
-	if race {
-		args = append(args, "-race")
-	}
-	args = append(args, target.buildPkg)
+	rmr(target.BinaryName())
+
+	args := []string{"build", "-i", "-v"}
+	args = appendParameters(args, tags, target)
 
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
 	runPrint("go", args...)
 }
 
+func appendParameters(args []string, tags []string, target target) []string {
+	if pkgdir != "" {
+		args = append(args, "-pkgdir", pkgdir)
+	}
+	if len(tags) > 0 {
+		args = append(args, "-tags", strings.Join(tags, " "))
+	}
+	if installSuffix != "" {
+		args = append(args, "-installsuffix", installSuffix)
+	}
+	if race {
+		args = append(args, "-race")
+	}
+
+	if !debugBinary {
+		// Regular binaries get version tagged and skip some debug symbols
+		args = append(args, "-ldflags", ldflags())
+	} else {
+		// -gcflags to disable optimizations and inlining. Skip -ldflags
+		// because `Could not launch program: decoding dwarf section info at
+		// offset 0x0: too short` on 'dlv exec ...' see
+		// https://github.com/derekparker/delve/issues/79
+		args = append(args, "-gcflags", "-N -l")
+	}
+
+	return append(args, target.buildPkg)
+}
+
 func buildTar(target target) {
 	name := archiveName(target)
 	filename := name + ".tar.gz"
@@ -483,22 +498,20 @@ func buildTar(target target) {
 	build(target, tags)
 
 	if goos == "darwin" {
-		macosCodesign(target.binaryName)
+		macosCodesign(target.BinaryName())
 	}
 
 	for i := range target.archiveFiles {
-		target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.binaryName, 1)
-		target.archiveFiles[i].dst = strings.Replace(target.archiveFiles[i].dst, "{{binary}}", target.binaryName, 1)
+		target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.BinaryName(), 1)
+		target.archiveFiles[i].dst = strings.Replace(target.archiveFiles[i].dst, "{{binary}}", target.BinaryName(), 1)
 		target.archiveFiles[i].dst = name + "/" + target.archiveFiles[i].dst
 	}
 
 	tarGz(filename, target.archiveFiles)
-	log.Println(filename)
+	fmt.Println(filename)
 }
 
 func buildZip(target target) {
-	target.binaryName += ".exe"
-
 	name := archiveName(target)
 	filename := name + ".zip"
 
@@ -510,14 +523,18 @@ func buildZip(target target) {
 
 	build(target, tags)
 
+	if goos == "windows" {
+		windowsCodesign(target.BinaryName())
+	}
+
 	for i := range target.archiveFiles {
-		target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.binaryName, 1)
-		target.archiveFiles[i].dst = strings.Replace(target.archiveFiles[i].dst, "{{binary}}", target.binaryName, 1)
+		target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.BinaryName(), 1)
+		target.archiveFiles[i].dst = strings.Replace(target.archiveFiles[i].dst, "{{binary}}", target.BinaryName(), 1)
 		target.archiveFiles[i].dst = name + "/" + target.archiveFiles[i].dst
 	}
 
 	zipFile(filename, target.archiveFiles)
-	log.Println(filename)
+	fmt.Println(filename)
 }
 
 func buildDeb(target target) {
@@ -537,8 +554,8 @@ func buildDeb(target target) {
 	build(target, []string{"noupgrade"})
 
 	for i := range target.installationFiles {
-		target.installationFiles[i].src = strings.Replace(target.installationFiles[i].src, "{{binary}}", target.binaryName, 1)
-		target.installationFiles[i].dst = strings.Replace(target.installationFiles[i].dst, "{{binary}}", target.binaryName, 1)
+		target.installationFiles[i].src = strings.Replace(target.installationFiles[i].src, "{{binary}}", target.BinaryName(), 1)
+		target.installationFiles[i].dst = strings.Replace(target.installationFiles[i].dst, "{{binary}}", target.BinaryName(), 1)
 	}
 
 	for _, af := range target.installationFiles {
@@ -551,16 +568,31 @@ func buildDeb(target target) {
 	debver := version
 	if strings.HasPrefix(debver, "v") {
 		debver = debver[1:]
+		// Debian interprets dashes as separator between main version and
+		// Debian package version, and thus thinks 0.14.26-rc.1 is better
+		// than just 0.14.26. This rectifies that.
+		debver = strings.Replace(debver, "-", "~", -1)
 	}
-	runPrint("fpm", "-t", "deb", "-s", "dir", "-C", "deb",
-		"-n", "syncthing", "-v", debver, "-a", debarch,
-		"--vendor", maintainer, "-m", maintainer,
-		"-d", "libc6",
-		"-d", "procps", // because postinst script
+	args := []string{
+		"-t", "deb",
+		"-s", "dir",
+		"-C", "deb",
+		"-n", target.debname,
+		"-v", debver,
+		"-a", debarch,
+		"-m", maintainer,
+		"--vendor", maintainer,
+		"--description", target.description,
 		"--url", "https://syncthing.net/",
-		"--description", "Open Source Continuous File Synchronization",
-		"--after-upgrade", "script/post-upgrade",
-		"--license", "MPL-2")
+		"--license", "MPL-2",
+	}
+	for _, dep := range target.debdeps {
+		args = append(args, "-d", dep)
+	}
+	if target.debpost != "" {
+		args = append(args, "--after-upgrade", target.debpost)
+	}
+	runPrint("fpm", args...)
 }
 
 func buildSnap(target target) {
@@ -585,7 +617,7 @@ func buildSnap(target target) {
 		snapver = snapver[1:]
 	}
 	snapgrade := "devel"
-	if matched, _ := regexp.MatchString(`^\d+\.\d+\.\d+$`, snapver); matched {
+	if matched, _ := regexp.MatchString(`^\d+\.\d+\.\d+(-rc.\d+)?$`, snapver); matched {
 		snapgrade = "stable"
 	}
 	err = tmpl.Execute(f, map[string]string{
@@ -601,21 +633,86 @@ func buildSnap(target target) {
 	runPrint("snapcraft")
 }
 
+func shouldBuildSyso(dir string) (string, error) {
+	jsonPath := filepath.Join(dir, "versioninfo.json")
+	file, err := os.Create(filepath.Join(dir, "versioninfo.json"))
+	if err != nil {
+		return "", errors.New("failed to create " + jsonPath + ": " + err.Error())
+	}
+
+	major, minor, patch, build := semanticVersion()
+	fmt.Fprintf(file, `{
+    "FixedFileInfo": {
+        "FileVersion": {
+            "Major": %s,
+            "Minor": %s,
+            "Patch": %s,
+            "Build": %s
+        }
+    },
+    "StringFileInfo": {
+        "FileDescription": "Open Source Continuous File Synchronization",
+        "LegalCopyright": "The Syncthing Authors",
+        "ProductVersion": "%s",
+        "ProductName": "Syncthing"
+    },
+    "IconPath": "assets/logo.ico"
+}`, major, minor, patch, build, getVersion())
+	file.Close()
+	defer func() {
+		if err := os.Remove(jsonPath); err != nil {
+			log.Printf("Warning: unable to remove generated %s: %v. Please remove it manually.", jsonPath, err)
+		}
+	}()
+
+	sysoPath := filepath.Join(dir, "cmd", "syncthing", "resource.syso")
+
+	if _, err := runError("goversioninfo", "-o", sysoPath); err != nil {
+		return "", errors.New("failed to create " + sysoPath + ": " + err.Error())
+	}
+
+	return sysoPath, nil
+}
+
+func shouldCleanupSyso(sysoFilePath string) {
+	if sysoFilePath == "" {
+		return
+	}
+	if err := os.Remove(sysoFilePath); err != nil {
+		log.Printf("Warning: unable to remove generated %s: %v. Please remove it manually.", sysoFilePath, err)
+	}
+}
+
+// copyFile copies a file from src to dst, ensuring the containing directory
+// exists. The permission bits are copied as well. If dst already exists and
+// the contents are identical to src the modification time is not updated.
 func copyFile(src, dst string, perm os.FileMode) error {
-	dstDir := filepath.Dir(dst)
-	os.MkdirAll(dstDir, 0755) // ignore error
-	srcFd, err := os.Open(src)
+	in, err := ioutil.ReadFile(src)
 	if err != nil {
 		return err
 	}
-	defer srcFd.Close()
-	dstFd, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm)
+
+	out, err := ioutil.ReadFile(dst)
 	if err != nil {
+		// The destination probably doesn't exist, we should create
+		// it.
+		goto copy
+	}
+
+	if bytes.Equal(in, out) {
+		// The permission bits may have changed without the contents
+		// changing so we always mirror them.
+		os.Chmod(dst, perm)
+		return nil
+	}
+
+copy:
+	os.MkdirAll(filepath.Dir(dst), 0777)
+	if err := ioutil.WriteFile(dst, in, perm); err != nil {
 		return err
 	}
-	defer dstFd.Close()
-	_, err = io.Copy(dstFd, srcFd)
-	return err
+
+	return nil
 }
 
 func listFiles(dir string) []string {
@@ -670,7 +767,7 @@ func shouldRebuildAssets(target, srcdir string) bool {
 }
 
 func proto() {
-	runPrint("go", "generate", "./lib/...")
+	runPrint("go", "generate", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/stdiscosrv")
 }
 
 func translate() {
@@ -762,6 +859,15 @@ func getVersion() string {
 	return "unknown-dev"
 }
 
+func semanticVersion() (major, minor, patch, build string) {
+	r := regexp.MustCompile(`v(?P<Major>\d+)\.(?P<Minor>\d+).(?P<Patch>\d+).*\+(?P<CommitsAhead>\d+)`)
+	matches := r.FindStringSubmatch(getVersion())
+	if len(matches) != 5 {
+		return "0", "0", "0", "0"
+	}
+	return matches[1], matches[2], matches[3], matches[4]
+}
+
 func getBranchSuffix() string {
 	bs, err := runError("git", "branch", "-a", "--contains")
 	if err != nil {
@@ -797,8 +903,9 @@ func getBranchSuffix() string {
 	}
 
 	branch = parts[len(parts)-1]
-	if branch == "master" {
-		// master builds are the default.
+	switch branch {
+	case "master", "release":
+		// these are not special
 		return ""
 	}
 
@@ -922,7 +1029,10 @@ func tarGz(out string, files []archiveFile) {
 		log.Fatal(err)
 	}
 
-	gw := gzip.NewWriter(fd)
+	gw, err := gzip.NewWriterLevel(fd, gzip.BestCompression)
+	if err != nil {
+		log.Fatal(err)
+	}
 	tw := tar.NewWriter(gw)
 
 	for _, f := range files {
@@ -975,6 +1085,21 @@ func zipFile(out string, files []archiveFile) {
 
 	zw := zip.NewWriter(fd)
 
+	var fw *flate.Writer
+
+	// Register the deflator.
+	zw.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
+		var err error
+		if fw == nil {
+			// Creating a flate compressor for every file is
+			// expensive, create one and reuse it.
+			fw, err = flate.NewWriter(out, flate.BestCompression)
+		} else {
+			fw.Reset(out)
+		}
+		return fw, err
+	})
+
 	for _, f := range files {
 		sf, err := os.Open(f.src)
 		if err != nil {
@@ -1050,59 +1175,166 @@ func macosCodesign(file string) {
 	}
 }
 
-func metalint(linters []string, dirs []string) {
-	ok := true
-	if isGometalinterInstalled() {
-		if !gometalinter(linters, dirs, lintExcludes...) {
-			ok = false
+func windowsCodesign(file string) {
+	st := "signtool.exe"
+
+	if path := os.Getenv("CODESIGN_SIGNTOOL"); path != "" {
+		st = path
+	}
+
+	for i, algo := range []string{"sha1", "sha256"} {
+		args := []string{"sign", "/fd", algo}
+		if f := os.Getenv("CODESIGN_CERTIFICATE_FILE"); f != "" {
+			args = append(args, "/f", f)
 		}
-	}
-	if !ok {
-		log.Fatal("Build succeeded, but there were lint warnings")
+		if p := os.Getenv("CODESIGN_CERTIFICATE_PASSWORD"); p != "" {
+			args = append(args, "/p", p)
+		}
+		if tr := os.Getenv("CODESIGN_TIMESTAMP_SERVER"); tr != "" {
+			switch algo {
+			case "sha256":
+				args = append(args, "/tr", tr, "/td", algo)
+			default:
+				args = append(args, "/t", tr)
+			}
+		}
+		if i > 0 {
+			args = append(args, "/as")
+		}
+		args = append(args, file)
+
+		bs, err := runError(st, args...)
+		if err != nil {
+			log.Println("Codesign: signing failed:", string(bs))
+			return
+		}
+		log.Println("Codesign: successfully signed", file, "using", algo)
 	}
 }
 
-func isGometalinterInstalled() bool {
-	if _, err := runError("gometalinter", "--disable-all"); err != nil {
-		log.Println("gometalinter is not installed")
-		return false
-	}
-	return true
+func metalint() {
+	lazyRebuildAssets()
+	runPrint("go", "test", "-run", "Metalint", "./meta")
 }
 
-func gometalinter(linters []string, dirs []string, excludes ...string) bool {
-	params := []string{"--disable-all", "--concurrency=2", "--deadline=300s"}
+func metalintShort() {
+	lazyRebuildAssets()
+	runPrint("go", "test", "-short", "-run", "Metalint", "./meta")
+}
 
-	for _, linter := range linters {
-		params = append(params, "--enable="+linter)
+func temporaryBuildDir() (string, error) {
+	// The base of our temp dir is "syncthing-xxxxxxxx" where the x:es
+	// are eight bytes from the sha256 of our working directory. We do
+	// this because we want a name in the global temp dir that doesn't
+	// conflict with someone else building syncthing on the same
+	// machine, yet is persistent between runs from the same source
+	// directory.
+	wd, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	hash := sha256.Sum256([]byte(wd))
+	base := fmt.Sprintf("syncthing-%x", hash[:4])
+
+	// The temp dir is taken from $STTMPDIR if set, otherwise the system
+	// default (potentially infrluenced by $TMPDIR on unixes).
+	var tmpDir string
+	if t := os.Getenv("STTMPDIR"); t != "" {
+		tmpDir = t
+	} else {
+		tmpDir = os.TempDir()
 	}
 
-	for _, exclude := range excludes {
-		params = append(params, "--exclude="+exclude)
+	return filepath.Join(tmpDir, base), nil
+}
+
+func buildGOPATH(gopath string) error {
+	pkg := filepath.Join(gopath, "src/github.com/syncthing/syncthing")
+	dirs := []string{"cmd", "lib", "meta", "script", "test", "vendor"}
+
+	if debug {
+		t0 := time.Now()
+		log.Println("build temporary GOPATH in", gopath)
+		defer func() {
+			log.Println("... in", time.Since(t0))
+		}()
 	}
 
+	// Walk the sources and copy the files into the temporary GOPATH.
+	// Remember which files are supposed to be present so we can clean
+	// out everything else in the next step. The copyFile() step will
+	// only actually copy the file if it doesn't exist or the contents
+	// differ.
+
+	exists := map[string]struct{}{}
 	for _, dir := range dirs {
-		params = append(params, dir)
+		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
+
+			dst := filepath.Join(pkg, path)
+			exists[dst] = struct{}{}
+
+			if err := copyFile(path, dst, info.Mode()); err != nil {
+				return err
+			}
+
+			return nil
+		})
+		if err != nil {
+			return err
+		}
 	}
 
-	bs, _ := runError("gometalinter", params...)
+	// Walk the temporary GOPATH and remove any files that we wouldn't
+	// have copied there in the previous step.
 
-	nerr := 0
-	lines := make(map[string]struct{})
-	for _, line := range strings.Split(string(bs), "\n") {
-		if line == "" {
-			continue
+	filepath.Walk(pkg, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
 		}
-		if _, ok := lines[line]; ok {
-			continue
+		if info.IsDir() {
+			return nil
 		}
-		log.Println(line)
-		if strings.Contains(line, "executable file not found") {
-			log.Println(` - Try "go run build.go setup" to install missing tools`)
+		if _, ok := exists[path]; !ok {
+			os.Remove(path)
 		}
-		lines[line] = struct{}{}
-		nerr++
-	}
+		return nil
+	})
 
-	return nerr == 0
+	return nil
+}
+
+func gopath() string {
+	if gopath := os.Getenv("GOPATH"); gopath != "" {
+		// The env var is set, use that.
+		return gopath
+	}
+
+	// Ask Go what it thinks.
+	bs, err := runError("go", "env", "GOPATH")
+	if err != nil {
+		return ""
+	}
+
+	// We got something. Check if we are in fact available in that location.
+	gopath := string(bs)
+	if _, err := os.Stat(filepath.Join(gopath, "src/github.com/syncthing/syncthing/build.go")); err == nil {
+		// That seems to be the gopath.
+		return gopath
+	}
+
+	// The gopath is not valid.
+	return ""
+}
+
+func (t target) BinaryName() string {
+	if goos == "windows" {
+		return t.binaryName + ".exe"
+	}
+	return t.binaryName
 }

build.sh

@@ -48,9 +48,6 @@ case "${1:-default}" in
 		;;
 
 	test)
-		ulimit -t 600 &>/dev/null || true
-		ulimit -d 512000 &>/dev/null || true
-		ulimit -m 512000 &>/dev/null || true
 		LOGGER_DISCARD=1 build test
 		;;
 
@@ -61,9 +58,9 @@ case "${1:-default}" in
 	prerelease)
 		go run script/authors.go
 		build transifex
-		git add -A gui/default/assets/ lib/auto/
 		pushd man ; ./refresh.sh ; popd
-		git add -A man
+		git add -A gui man
+		git commit -m 'gui, man: Update docs & translations'
 		;;
 
 	noupgrade)
@@ -93,44 +90,7 @@ case "${1:-default}" in
 		done
 		;;
 
-	test-cov)
-		ulimit -t 600 &>/dev/null || true
-		ulimit -d 512000 &>/dev/null || true
-		ulimit -m 512000 &>/dev/null || true
-
-		echo "mode: set" > coverage.out
-		fail=0
-
-		# For every package in the repo
-		for dir in $(go list ./lib/... ./cmd/...) ; do
-			# run the tests
-			GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -coverprofile=profile.out $dir
-			if [ -f profile.out ] ; then
-				# and if there was test output, append it to coverage.out
-				grep -v "mode: " profile.out >> coverage.out
-				rm profile.out
-			fi
-		done
-
-		notCovered=$(egrep -c '\s0$' coverage.out)
-		total=$(wc -l coverage.out | awk '{print $1}')
-		coverPct=$(awk "BEGIN{print (1 - $notCovered / $total) * 100}")
-		echo "Total coverage is $coverPct%"
-
-		gocov convert coverage.out | gocov-xml > coverage.xml
-
-		# This is usually run from within Jenkins. If it is, we need to
-		# tweak the paths in coverage.xml so cobertura finds the
-		# source.
-		if [[ "${WORKSPACE:-default}" != "default" ]] ; then
-			sed "s#$WORKSPACE##g" < coverage.xml > coverage.xml.new && mv coverage.xml.new coverage.xml
-		fi
-		;;
-
 	test-xunit)
-		ulimit -t 600 &>/dev/null || true
-		ulimit -d 512000 &>/dev/null || true
-		ulimit -m 512000 &>/dev/null || true
 
 		(GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -v -race ./lib/... ./cmd/... || true) > tests.out
 		go2xunit -output tests.xml -fail < tests.out

cmd/stbench/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 // This doesn't build on Windows due to the Rusage stuff.
 
@@ -92,7 +92,7 @@ loop:
 	return t1.Sub(t0)
 }
 
-// report stops the given process and reports on it's resource usage in two
+// report stops the given process and reports on its resource usage in two
 // ways: human readable to stderr, and CSV to stdout.
 func report(p *rc.Process, wallTime time.Duration) {
 	sv, err := p.SystemVersion()

cmd/stcli/README.md

@@ -1,10 +0,0 @@
-syncthing-cli
-=============
-
-[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing-cli.svg?style=flat-square)](http://build.syncthing.net/job/syncthing-cli/lastBuild/)
-
-A CLI that talks to the Syncthing REST interface.
-
-`go get github.com/syncthing/syncthing-cli`
-
-Or download the [latest build](http://build.syncthing.net/job/syncthing-cli/lastSuccessfulBuild/artifact/).

cmd/stcli/cmd_folders.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/AudriusButkevicius/cli"
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/fs"
 )
 
 func init() {
@@ -102,8 +103,10 @@ func foldersList(c *cli.Context) {
 		if !first {
 			fmt.Fprintln(writer)
 		}
+		fs := folder.Filesystem()
 		fmt.Fprintln(writer, "ID:\t", folder.ID, "\t")
-		fmt.Fprintln(writer, "Path:\t", folder.RawPath, "\t(directory)")
+		fmt.Fprintln(writer, "Path:\t", fs.URI(), "\t(directory)")
+		fmt.Fprintln(writer, "Path type:\t", fs.Type(), "\t(directory-type)")
 		fmt.Fprintln(writer, "Folder type:\t", folder.Type, "\t(type)")
 		fmt.Fprintln(writer, "Ignore permissions:\t", folder.IgnorePerms, "\t(permissions)")
 		fmt.Fprintln(writer, "Rescan interval in seconds:\t", folder.RescanIntervalS, "\t(rescan)")
@@ -124,8 +127,9 @@ func foldersAdd(c *cli.Context) {
 	abs, err := filepath.Abs(c.Args()[1])
 	die(err)
 	folder := config.FolderConfiguration{
-		ID:      c.Args()[0],
-		RawPath: filepath.Clean(abs),
+		ID:             c.Args()[0],
+		Path:           filepath.Clean(abs),
+		FilesystemType: fs.FilesystemTypeBasic,
 	}
 	cfg.Folders = append(cfg.Folders, folder)
 	setConfig(c, cfg)
@@ -185,7 +189,9 @@ func foldersGet(c *cli.Context) {
 		}
 		switch arg {
 		case "directory":
-			fmt.Println(folder.RawPath)
+			fmt.Println(folder.Filesystem().URI())
+		case "directory-type":
+			fmt.Println(folder.Filesystem().Type())
 		case "type":
 			fmt.Println(folder.Type)
 		case "permissions":
@@ -197,7 +203,7 @@ func foldersGet(c *cli.Context) {
 				fmt.Println(folder.Versioning.Type)
 			}
 		default:
-			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, type, permissions, versioning, versioning-<key>")
+			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, directory-type, type, permissions, versioning, versioning-<key>")
 		}
 		return
 	}
@@ -220,7 +226,11 @@ func foldersSet(c *cli.Context) {
 		}
 		switch arg {
 		case "directory":
-			cfg.Folders[i].RawPath = val
+			cfg.Folders[i].Path = val
+		case "directory-type":
+			var fsType fs.FilesystemType
+			fsType.UnmarshalText([]byte(val))
+			cfg.Folders[i].FilesystemType = fsType
 		case "type":
 			var t config.FolderType
 			if err := t.UnmarshalText([]byte(val)); err != nil {

cmd/stcompdirs/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -15,8 +15,6 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-
-	"github.com/syncthing/syncthing/lib/symlinks"
 )
 
 func main() {
@@ -90,10 +88,10 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 		}
 
 		rn, _ := filepath.Rel(dir, path)
-		if rn == "." || rn == ".stfolder" {
+		if rn == "." {
 			return nil
 		}
-		if rn == ".stversions" {
+		if rn == ".stversions" || rn == ".stfolder" {
 			return filepath.SkipDir
 		}
 
@@ -104,7 +102,7 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 				mode: os.ModeSymlink,
 			}
 
-			tgt, _, err := symlinks.Read(path)
+			tgt, err := os.Readlink(path)
 			if err != nil {
 				return err
 			}

cmd/stdisco/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/stdiscosrv/LICENSE

@@ -1,19 +0,0 @@
-Copyright (C) 2014-2015 The Discosrv Authors
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-- The above copyright notice and this permission notice shall be included in
-  all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

cmd/stdiscosrv/README.md

@@ -1,44 +1,10 @@
 stdiscosrv
 ==========
 
-[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/stdiscosrv.svg?style=flat-square)](http://build.syncthing.net/job/stdiscosrv/lastBuild/)
-
 This is the global discovery server for the `syncthing` project.
 
-To get it, run `go get github.com/syncthing/stdiscosrv` or download the
-[latest build](http://build.syncthing.net/job/stdiscosrv/lastSuccessfulBuild/artifact/)
-from the build server.
-
 Usage
 -----
 
-The discovery server supports `ql` and `postgres` backends.
-Specify the backend via `-db-backend` and the database DSN via `-db-dsn`.
+https://docs.syncthing.net/users/stdiscosrv.html
 
-By default it will use in-memory `ql` backend. If you wish to persist the
-information on disk between restarts in `ql`, specify a file DSN:
-
-```bash
-$ stdiscosrv -db-dsn="file:///var/run/stdiscosrv.db"
-```
-
-For `postgres`, you will need to create a database and a user with permissions
-to create tables in it, then start the stdiscosrv as follows:
-
-```bash
-$ export STDISCOSRV_DB_DSN="postgres://user:password@localhost/databasename"
-$ stdiscosrv -db-backend="postgres"
-```
-
-You can pass the DSN as command line option, but the value what you pass in will
-be visible in most process managers, potentially exposing the database password
-to other users.
-
-In all cases, the appropriate tables and indexes will be created at first
-startup. If it doesn't exit with an error, you're fine.
-
-See `stdiscosrv -help` for other options.
-
-##### Third-party attribution
-
-[cznic/lldb](https://github.com/cznic/lldb), Copyright (C) 2014 The lldb Authors.

cmd/stdiscosrv/apisrv.go

@@ -0,0 +1,397 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"log"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+// announcement is the format received from and sent to clients
+type announcement struct {
+	Seen      time.Time `json:"seen"`
+	Addresses []string  `json:"addresses"`
+}
+
+type apiSrv struct {
+	addr     string
+	cert     tls.Certificate
+	db       database
+	listener net.Listener
+	repl     replicator // optional
+	useHTTP  bool
+
+	mapsMut sync.Mutex
+	misses  map[string]int32
+}
+
+type requestID int64
+
+func (i requestID) String() string {
+	return fmt.Sprintf("%016x", int64(i))
+}
+
+type contextKey int
+
+const idKey contextKey = iota
+
+func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP bool) *apiSrv {
+	return &apiSrv{
+		addr:    addr,
+		cert:    cert,
+		db:      db,
+		repl:    repl,
+		useHTTP: useHTTP,
+		misses:  make(map[string]int32),
+	}
+}
+
+func (s *apiSrv) Serve() {
+	if s.useHTTP {
+		listener, err := net.Listen("tcp", s.addr)
+		if err != nil {
+			log.Println("Listen:", err)
+			return
+		}
+		s.listener = listener
+	} else {
+		tlsCfg := &tls.Config{
+			Certificates:           []tls.Certificate{s.cert},
+			ClientAuth:             tls.RequestClientCert,
+			SessionTicketsDisabled: true,
+			MinVersion:             tls.VersionTLS12,
+			CipherSuites: []uint16{
+				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+			},
+		}
+
+		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
+		if err != nil {
+			log.Println("Listen:", err)
+			return
+		}
+		s.listener = tlsListener
+	}
+
+	http.HandleFunc("/", s.handler)
+	http.HandleFunc("/ping", handlePing)
+
+	srv := &http.Server{
+		ReadTimeout:    httpReadTimeout,
+		WriteTimeout:   httpWriteTimeout,
+		MaxHeaderBytes: httpMaxHeaderBytes,
+	}
+
+	if err := srv.Serve(s.listener); err != nil {
+		log.Println("Serve:", err)
+	}
+}
+
+var topCtx = context.Background()
+
+func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
+	t0 := time.Now()
+
+	lw := NewLoggingResponseWriter(w)
+
+	defer func() {
+		diff := time.Since(t0)
+		apiRequestsSeconds.WithLabelValues(req.Method).Observe(diff.Seconds())
+		apiRequestsTotal.WithLabelValues(req.Method, strconv.Itoa(lw.statusCode)).Inc()
+	}()
+
+	reqID := requestID(rand.Int63())
+	ctx := context.WithValue(topCtx, idKey, reqID)
+
+	if debug {
+		log.Println(reqID, req.Method, req.URL)
+	}
+
+	var remoteIP net.IP
+	if s.useHTTP {
+		remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
+	} else {
+		addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
+		if err != nil {
+			log.Println("remoteAddr:", err)
+			lw.Header().Set("Retry-After", errorRetryAfterString())
+			http.Error(lw, "Internal Server Error", http.StatusInternalServerError)
+			apiRequestsTotal.WithLabelValues("no_remote_addr").Inc()
+			return
+		}
+		remoteIP = addr.IP
+	}
+
+	switch req.Method {
+	case "GET":
+		s.handleGET(ctx, lw, req)
+	case "POST":
+		s.handlePOST(ctx, remoteIP, lw, req)
+	default:
+		http.Error(lw, "Method Not Allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *apiSrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
+	reqID := ctx.Value(idKey).(requestID)
+
+	deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
+	if err != nil {
+		if debug {
+			log.Println(reqID, "bad device param")
+		}
+		lookupRequestsTotal.WithLabelValues("bad_request").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	key := deviceID.String()
+	rec, err := s.db.get(key)
+	if err != nil {
+		// some sort of internal error
+		lookupRequestsTotal.WithLabelValues("internal_error").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+
+	if len(rec.Addresses) == 0 {
+		lookupRequestsTotal.WithLabelValues("not_found").Inc()
+
+		s.mapsMut.Lock()
+		misses := s.misses[key]
+		if misses < rec.Misses {
+			misses = rec.Misses + 1
+		} else {
+			misses++
+		}
+		s.misses[key] = misses
+		s.mapsMut.Unlock()
+
+		if misses%notFoundMissesWriteInterval == 0 {
+			rec.Misses = misses
+			rec.Missed = time.Now().UnixNano()
+			rec.Addresses = nil
+			// rec.Seen retained from get
+			s.db.put(key, rec)
+		}
+
+		w.Header().Set("Retry-After", notFoundRetryAfterString(int(misses)))
+		http.Error(w, "Not Found", http.StatusNotFound)
+		return
+	}
+
+	lookupRequestsTotal.WithLabelValues("success").Inc()
+
+	bs, _ := json.Marshal(announcement{
+		Seen:      time.Unix(0, rec.Seen),
+		Addresses: addressStrs(rec.Addresses),
+	})
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bs)
+}
+
+func (s *apiSrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
+	reqID := ctx.Value(idKey).(requestID)
+
+	rawCert := certificateBytes(req)
+	if rawCert == nil {
+		if debug {
+			log.Println(reqID, "no certificates")
+		}
+		announceRequestsTotal.WithLabelValues("no_certificate").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Forbidden", http.StatusForbidden)
+		return
+	}
+
+	var ann announcement
+	if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
+		if debug {
+			log.Println(reqID, "decode:", err)
+		}
+		announceRequestsTotal.WithLabelValues("bad_request").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	deviceID := protocol.NewDeviceID(rawCert)
+
+	addresses := fixupAddresses(remoteIP, ann.Addresses)
+	if len(addresses) == 0 {
+		announceRequestsTotal.WithLabelValues("bad_request").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	if err := s.handleAnnounce(remoteIP, deviceID, addresses); err != nil {
+		announceRequestsTotal.WithLabelValues("internal_error").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+
+	announceRequestsTotal.WithLabelValues("success").Inc()
+
+	w.Header().Set("Reannounce-After", reannounceAfterString())
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func (s *apiSrv) Stop() {
+	s.listener.Close()
+}
+
+func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addresses []string) error {
+	key := deviceID.String()
+	now := time.Now()
+	expire := now.Add(addressExpiryTime).UnixNano()
+
+	dbAddrs := make([]DatabaseAddress, len(addresses))
+	for i := range addresses {
+		dbAddrs[i].Address = addresses[i]
+		dbAddrs[i].Expires = expire
+	}
+
+	seen := now.UnixNano()
+	if s.repl != nil {
+		s.repl.send(key, dbAddrs, seen)
+	}
+	return s.db.merge(key, dbAddrs, seen)
+}
+
+func handlePing(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(204)
+}
+
+func certificateBytes(req *http.Request) []byte {
+	if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
+		return req.TLS.PeerCertificates[0].Raw
+	}
+
+	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
+		bs := []byte(hdr)
+		// The certificate is in PEM format but with spaces for newlines. We
+		// need to reinstate the newlines for the PEM decoder. But we need to
+		// leave the spaces in the BEGIN and END lines - the first and last
+		// space - alone.
+		firstSpace := bytes.Index(bs, []byte(" "))
+		lastSpace := bytes.LastIndex(bs, []byte(" "))
+		for i := firstSpace + 1; i < lastSpace; i++ {
+			if bs[i] == ' ' {
+				bs[i] = '\n'
+			}
+		}
+		block, _ := pem.Decode(bs)
+		if block == nil {
+			// Decoding failed
+			return nil
+		}
+		return block.Bytes
+	}
+
+	return nil
+}
+
+// fixupAddresses checks the list of addresses, removing invalid ones and
+// replacing unspecified IPs with the given remote IP.
+func fixupAddresses(remote net.IP, addresses []string) []string {
+	fixed := make([]string, 0, len(addresses))
+	for _, annAddr := range addresses {
+		uri, err := url.Parse(annAddr)
+		if err != nil {
+			continue
+		}
+
+		host, port, err := net.SplitHostPort(uri.Host)
+		if err != nil {
+			continue
+		}
+
+		ip := net.ParseIP(host)
+		if host == "" || ip.IsUnspecified() {
+			// Do not use IPv6 remote address if requested scheme is ...4
+			// (i.e., tcp4, etc.)
+			if strings.HasSuffix(uri.Scheme, "4") && remote.To4() == nil {
+				continue
+			}
+
+			// Do not use IPv4 remote address if requested scheme is ...6
+			if strings.HasSuffix(uri.Scheme, "6") && remote.To4() != nil {
+				continue
+			}
+
+			host = remote.String()
+		}
+
+		uri.Host = net.JoinHostPort(host, port)
+		fixed = append(fixed, uri.String())
+	}
+
+	return fixed
+}
+
+type loggingResponseWriter struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
+	return &loggingResponseWriter{w, http.StatusOK}
+}
+
+func (lrw *loggingResponseWriter) WriteHeader(code int) {
+	lrw.statusCode = code
+	lrw.ResponseWriter.WriteHeader(code)
+}
+
+func addressStrs(dbAddrs []DatabaseAddress) []string {
+	res := make([]string, len(dbAddrs))
+	for i, a := range dbAddrs {
+		res[i] = a.Address
+	}
+	return res
+}
+
+func errorRetryAfterString() string {
+	return strconv.Itoa(errorRetryAfterSeconds + rand.Intn(errorRetryFuzzSeconds))
+}
+
+func notFoundRetryAfterString(misses int) string {
+	retryAfterS := notFoundRetryMinSeconds + notFoundRetryIncSeconds*misses
+	if retryAfterS > notFoundRetryMaxSeconds {
+		retryAfterS = notFoundRetryMaxSeconds
+	}
+	retryAfterS += rand.Intn(notFoundRetryFuzzSeconds)
+	return strconv.Itoa(retryAfterS)
+}
+
+func reannounceAfterString() string {
+	return strconv.Itoa(reannounceAfterSeconds + rand.Intn(reannounzeFuzzSeconds))
+}

cmd/stdiscosrv/clean.go

@@ -1,75 +0,0 @@
-// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-
-package main
-
-import (
-	"database/sql"
-	"log"
-	"time"
-)
-
-type cleansrv struct {
-	intv time.Duration
-	db   *sql.DB
-	prep map[string]*sql.Stmt
-}
-
-func (s *cleansrv) Serve() {
-	for {
-		time.Sleep(next(s.intv))
-
-		err := s.cleanOldEntries()
-		if err != nil {
-			log.Println("Clean:", err)
-		}
-	}
-}
-
-func (s *cleansrv) Stop() {
-	panic("stop unimplemented")
-}
-
-func (s *cleansrv) cleanOldEntries() (err error) {
-	var tx *sql.Tx
-	tx, err = s.db.Begin()
-	if err != nil {
-		return err
-	}
-
-	defer func() {
-		if err == nil {
-			err = tx.Commit()
-		} else {
-			tx.Rollback()
-		}
-	}()
-
-	res, err := tx.Stmt(s.prep["cleanAddress"]).Exec()
-	if err != nil {
-		return err
-	}
-	if rows, _ := res.RowsAffected(); rows > 0 {
-		log.Printf("Clean: %d old addresses", rows)
-	}
-
-	res, err = tx.Stmt(s.prep["cleanDevice"]).Exec()
-	if err != nil {
-		return err
-	}
-	if rows, _ := res.RowsAffected(); rows > 0 {
-		log.Printf("Clean: %d old devices", rows)
-	}
-
-	var devs, addrs int
-	row := tx.Stmt(s.prep["countDevice"]).QueryRow()
-	if err = row.Scan(&devs); err != nil {
-		return err
-	}
-	row = tx.Stmt(s.prep["countAddress"]).QueryRow()
-	if err = row.Scan(&addrs); err != nil {
-		return err
-	}
-
-	log.Printf("Database: %d devices, %d addresses", devs, addrs)
-	return nil
-}

cmd/stdiscosrv/database.go

@@ -0,0 +1,336 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//go:generate go run ../../script/protofmt.go database.proto
+//go:generate protoc -I ../../../../../ -I ../../vendor/ -I ../../vendor/github.com/gogo/protobuf/protobuf -I . --gogofast_out=. database.proto
+
+package main
+
+import (
+	"sort"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+type clock interface {
+	Now() time.Time
+}
+
+type defaultClock struct{}
+
+func (defaultClock) Now() time.Time {
+	return time.Now()
+}
+
+type database interface {
+	put(key string, rec DatabaseRecord) error
+	merge(key string, addrs []DatabaseAddress, seen int64) error
+	get(key string) (DatabaseRecord, error)
+}
+
+type levelDBStore struct {
+	db         *leveldb.DB
+	inbox      chan func()
+	stop       chan struct{}
+	clock      clock
+	marshalBuf []byte
+}
+
+func newLevelDBStore(dir string) (*levelDBStore, error) {
+	db, err := leveldb.OpenFile(dir, levelDBOptions)
+	if err != nil {
+		return nil, err
+	}
+	return &levelDBStore{
+		db:    db,
+		inbox: make(chan func(), 16),
+		stop:  make(chan struct{}),
+		clock: defaultClock{},
+	}, nil
+}
+
+func (s *levelDBStore) put(key string, rec DatabaseRecord) error {
+	t0 := time.Now()
+	defer func() {
+		databaseOperationSeconds.WithLabelValues(dbOpPut).Observe(time.Since(t0).Seconds())
+	}()
+
+	rc := make(chan error)
+
+	s.inbox <- func() {
+		size := rec.Size()
+		if len(s.marshalBuf) < size {
+			s.marshalBuf = make([]byte, size)
+		}
+		n, _ := rec.MarshalTo(s.marshalBuf)
+		rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
+	}
+
+	err := <-rc
+	if err != nil {
+		databaseOperations.WithLabelValues(dbOpPut, dbResError).Inc()
+	} else {
+		databaseOperations.WithLabelValues(dbOpPut, dbResSuccess).Inc()
+	}
+
+	return err
+}
+
+func (s *levelDBStore) merge(key string, addrs []DatabaseAddress, seen int64) error {
+	t0 := time.Now()
+	defer func() {
+		databaseOperationSeconds.WithLabelValues(dbOpMerge).Observe(time.Since(t0).Seconds())
+	}()
+
+	rc := make(chan error)
+	newRec := DatabaseRecord{
+		Addresses: addrs,
+		Seen:      seen,
+	}
+
+	s.inbox <- func() {
+		// grab the existing record
+		oldRec, err := s.get(key)
+		if err != nil {
+			// "not found" is not an error from get, so this is serious
+			// stuff only
+			rc <- err
+			return
+		}
+		newRec = merge(newRec, oldRec)
+
+		// We replicate s.put() functionality here ourselves instead of
+		// calling it because we want to serialize our get above together
+		// with the put in the same function.
+		size := newRec.Size()
+		if len(s.marshalBuf) < size {
+			s.marshalBuf = make([]byte, size)
+		}
+		n, _ := newRec.MarshalTo(s.marshalBuf)
+		rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
+	}
+
+	err := <-rc
+	if err != nil {
+		databaseOperations.WithLabelValues(dbOpMerge, dbResError).Inc()
+	} else {
+		databaseOperations.WithLabelValues(dbOpMerge, dbResSuccess).Inc()
+	}
+
+	return err
+}
+
+func (s *levelDBStore) get(key string) (DatabaseRecord, error) {
+	t0 := time.Now()
+	defer func() {
+		databaseOperationSeconds.WithLabelValues(dbOpGet).Observe(time.Since(t0).Seconds())
+	}()
+
+	keyBs := []byte(key)
+	val, err := s.db.Get(keyBs, nil)
+	if err == leveldb.ErrNotFound {
+		databaseOperations.WithLabelValues(dbOpGet, dbResNotFound).Inc()
+		return DatabaseRecord{}, nil
+	}
+	if err != nil {
+		databaseOperations.WithLabelValues(dbOpGet, dbResError).Inc()
+		return DatabaseRecord{}, err
+	}
+
+	var rec DatabaseRecord
+
+	if err := rec.Unmarshal(val); err != nil {
+		databaseOperations.WithLabelValues(dbOpGet, dbResUnmarshalError).Inc()
+		return DatabaseRecord{}, nil
+	}
+
+	rec.Addresses = expire(rec.Addresses, s.clock.Now().UnixNano())
+	databaseOperations.WithLabelValues(dbOpGet, dbResSuccess).Inc()
+	return rec, nil
+}
+
+func (s *levelDBStore) Serve() {
+	t := time.NewTimer(0)
+	defer t.Stop()
+	defer s.db.Close()
+
+	// Start the statistics serve routine. It will exit with us when
+	// statisticsTrigger is closed.
+	statisticsTrigger := make(chan struct{})
+	defer close(statisticsTrigger)
+	statisticsDone := make(chan struct{})
+	go s.statisticsServe(statisticsTrigger, statisticsDone)
+
+	for {
+		select {
+		case fn := <-s.inbox:
+			// Run function in serialized order.
+			fn()
+
+		case <-t.C:
+			// Trigger the statistics routine to do its thing in the
+			// background.
+			statisticsTrigger <- struct{}{}
+
+		case <-statisticsDone:
+			// The statistics routine is done with one iteratation, schedule
+			// the next.
+			t.Reset(databaseStatisticsInterval)
+
+		case <-s.stop:
+			// We're done.
+			return
+		}
+	}
+}
+
+func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
+	for range trigger {
+		t0 := time.Now()
+		nowNanos := t0.UnixNano()
+		cutoff24h := t0.Add(-24 * time.Hour).UnixNano()
+		cutoff1w := t0.Add(-7 * 24 * time.Hour).UnixNano()
+		current, last24h, last1w, inactive, errors := 0, 0, 0, 0, 0
+
+		iter := s.db.NewIterator(&util.Range{}, nil)
+		for iter.Next() {
+			// Attempt to unmarshal the record and count the
+			// failure if there's something wrong with it.
+			var rec DatabaseRecord
+			if err := rec.Unmarshal(iter.Value()); err != nil {
+				errors++
+				continue
+			}
+
+			// If there are addresses that have not expired it's a current
+			// record, otherwise account it based on when it was last seen
+			// (last 24 hours or last week) or finally as inactice.
+			switch {
+			case len(expire(rec.Addresses, nowNanos)) > 0:
+				current++
+			case rec.Seen > cutoff24h:
+				last24h++
+			case rec.Seen > cutoff1w:
+				last1w++
+			default:
+				inactive++
+			}
+		}
+
+		iter.Release()
+
+		databaseKeys.WithLabelValues("current").Set(float64(current))
+		databaseKeys.WithLabelValues("last24h").Set(float64(last24h))
+		databaseKeys.WithLabelValues("last1w").Set(float64(last1w))
+		databaseKeys.WithLabelValues("inactive").Set(float64(inactive))
+		databaseKeys.WithLabelValues("error").Set(float64(errors))
+		databaseStatisticsSeconds.Set(time.Since(t0).Seconds())
+
+		// Signal that we are done and can be scheduled again.
+		done <- struct{}{}
+	}
+}
+
+func (s *levelDBStore) Stop() {
+	close(s.stop)
+}
+
+// merge returns the merged result of the two database records a and b. The
+// result is the union of the two address sets, with the newer expiry time
+// chosen for any duplicates.
+func merge(a, b DatabaseRecord) DatabaseRecord {
+	// Both lists must be sorted for this to work.
+	sort.Slice(a.Addresses, func(i, j int) bool {
+		return a.Addresses[i].Address < a.Addresses[j].Address
+	})
+	sort.Slice(b.Addresses, func(i, j int) bool {
+		return b.Addresses[i].Address < b.Addresses[j].Address
+	})
+
+	res := DatabaseRecord{
+		Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
+		Seen:      a.Seen,
+	}
+	if b.Seen > a.Seen {
+		res.Seen = b.Seen
+	}
+
+	aIdx := 0
+	bIdx := 0
+	aAddrs := a.Addresses
+	bAddrs := b.Addresses
+loop:
+	for {
+		switch {
+		case aIdx == len(aAddrs) && bIdx == len(bAddrs):
+			// both lists are exhausted, we are done
+			break loop
+
+		case aIdx == len(aAddrs):
+			// a is exhausted, pick from b and continue
+			res.Addresses = append(res.Addresses, bAddrs[bIdx])
+			bIdx++
+			continue
+
+		case bIdx == len(bAddrs):
+			// b is exhausted, pick from a and continue
+			res.Addresses = append(res.Addresses, aAddrs[aIdx])
+			aIdx++
+			continue
+		}
+
+		// We have values left on both sides.
+		aVal := aAddrs[aIdx]
+		bVal := bAddrs[bIdx]
+
+		switch {
+		case aVal.Address == bVal.Address:
+			// update for same address, pick newer
+			if aVal.Expires > bVal.Expires {
+				res.Addresses = append(res.Addresses, aVal)
+			} else {
+				res.Addresses = append(res.Addresses, bVal)
+			}
+			aIdx++
+			bIdx++
+
+		case aVal.Address < bVal.Address:
+			// a is smallest, pick it and continue
+			res.Addresses = append(res.Addresses, aVal)
+			aIdx++
+
+		default:
+			// b is smallest, pick it and continue
+			res.Addresses = append(res.Addresses, bVal)
+			bIdx++
+		}
+	}
+	return res
+}
+
+// expire returns the list of addresses after removing expired entries.
+// Expiration happen in place, so the slice given as the parameter is
+// destroyed. Internal order is not preserved.
+func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
+	i := 0
+	for i < len(addrs) {
+		if addrs[i].Expires < now {
+			// This item is expired. Replace it with the last in the list
+			// (noop if we are at the last item).
+			addrs[i] = addrs[len(addrs)-1]
+			// Wipe the last item of the list to release references to
+			// strings and stuff.
+			addrs[len(addrs)-1] = DatabaseAddress{}
+			// Shorten the slice.
+			addrs = addrs[:len(addrs)-1]
+			continue
+		}
+		i++
+	}
+	return addrs
+}

cmd/stdiscosrv/database.pb.go

@@ -0,0 +1,772 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: database.proto
+
+/*
+	Package main is a generated protocol buffer package.
+
+	It is generated from these files:
+		database.proto
+
+	It has these top-level messages:
+		DatabaseRecord
+		ReplicationRecord
+		DatabaseAddress
+*/
+package main
+
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import _ "github.com/gogo/protobuf/gogoproto"
+
+import io "io"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+
+type DatabaseRecord struct {
+	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses" json:"addresses"`
+	Misses    int32             `protobuf:"varint,2,opt,name=misses,proto3" json:"misses,omitempty"`
+	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
+	Missed    int64             `protobuf:"varint,4,opt,name=missed,proto3" json:"missed,omitempty"`
+}
+
+func (m *DatabaseRecord) Reset()                    { *m = DatabaseRecord{} }
+func (m *DatabaseRecord) String() string            { return proto.CompactTextString(m) }
+func (*DatabaseRecord) ProtoMessage()               {}
+func (*DatabaseRecord) Descriptor() ([]byte, []int) { return fileDescriptorDatabase, []int{0} }
+
+type ReplicationRecord struct {
+	Key       string            `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+	Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses" json:"addresses"`
+	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
+}
+
+func (m *ReplicationRecord) Reset()                    { *m = ReplicationRecord{} }
+func (m *ReplicationRecord) String() string            { return proto.CompactTextString(m) }
+func (*ReplicationRecord) ProtoMessage()               {}
+func (*ReplicationRecord) Descriptor() ([]byte, []int) { return fileDescriptorDatabase, []int{1} }
+
+type DatabaseAddress struct {
+	Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
+	Expires int64  `protobuf:"varint,2,opt,name=expires,proto3" json:"expires,omitempty"`
+}
+
+func (m *DatabaseAddress) Reset()                    { *m = DatabaseAddress{} }
+func (m *DatabaseAddress) String() string            { return proto.CompactTextString(m) }
+func (*DatabaseAddress) ProtoMessage()               {}
+func (*DatabaseAddress) Descriptor() ([]byte, []int) { return fileDescriptorDatabase, []int{2} }
+
+func init() {
+	proto.RegisterType((*DatabaseRecord)(nil), "main.DatabaseRecord")
+	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
+	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
+}
+func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Addresses) > 0 {
+		for _, msg := range m.Addresses {
+			dAtA[i] = 0xa
+			i++
+			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
+			n, err := msg.MarshalTo(dAtA[i:])
+			if err != nil {
+				return 0, err
+			}
+			i += n
+		}
+	}
+	if m.Misses != 0 {
+		dAtA[i] = 0x10
+		i++
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+	}
+	if m.Seen != 0 {
+		dAtA[i] = 0x18
+		i++
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+	}
+	if m.Missed != 0 {
+		dAtA[i] = 0x20
+		i++
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+	}
+	return i, nil
+}
+
+func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Key) > 0 {
+		dAtA[i] = 0xa
+		i++
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		i += copy(dAtA[i:], m.Key)
+	}
+	if len(m.Addresses) > 0 {
+		for _, msg := range m.Addresses {
+			dAtA[i] = 0x12
+			i++
+			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
+			n, err := msg.MarshalTo(dAtA[i:])
+			if err != nil {
+				return 0, err
+			}
+			i += n
+		}
+	}
+	if m.Seen != 0 {
+		dAtA[i] = 0x18
+		i++
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+	}
+	return i, nil
+}
+
+func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Address) > 0 {
+		dAtA[i] = 0xa
+		i++
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
+		i += copy(dAtA[i:], m.Address)
+	}
+	if m.Expires != 0 {
+		dAtA[i] = 0x10
+		i++
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
+	}
+	return i, nil
+}
+
+func encodeFixed64Database(dAtA []byte, offset int, v uint64) int {
+	dAtA[offset] = uint8(v)
+	dAtA[offset+1] = uint8(v >> 8)
+	dAtA[offset+2] = uint8(v >> 16)
+	dAtA[offset+3] = uint8(v >> 24)
+	dAtA[offset+4] = uint8(v >> 32)
+	dAtA[offset+5] = uint8(v >> 40)
+	dAtA[offset+6] = uint8(v >> 48)
+	dAtA[offset+7] = uint8(v >> 56)
+	return offset + 8
+}
+func encodeFixed32Database(dAtA []byte, offset int, v uint32) int {
+	dAtA[offset] = uint8(v)
+	dAtA[offset+1] = uint8(v >> 8)
+	dAtA[offset+2] = uint8(v >> 16)
+	dAtA[offset+3] = uint8(v >> 24)
+	return offset + 4
+}
+func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return offset + 1
+}
+func (m *DatabaseRecord) Size() (n int) {
+	var l int
+	_ = l
+	if len(m.Addresses) > 0 {
+		for _, e := range m.Addresses {
+			l = e.Size()
+			n += 1 + l + sovDatabase(uint64(l))
+		}
+	}
+	if m.Misses != 0 {
+		n += 1 + sovDatabase(uint64(m.Misses))
+	}
+	if m.Seen != 0 {
+		n += 1 + sovDatabase(uint64(m.Seen))
+	}
+	if m.Missed != 0 {
+		n += 1 + sovDatabase(uint64(m.Missed))
+	}
+	return n
+}
+
+func (m *ReplicationRecord) Size() (n int) {
+	var l int
+	_ = l
+	l = len(m.Key)
+	if l > 0 {
+		n += 1 + l + sovDatabase(uint64(l))
+	}
+	if len(m.Addresses) > 0 {
+		for _, e := range m.Addresses {
+			l = e.Size()
+			n += 1 + l + sovDatabase(uint64(l))
+		}
+	}
+	if m.Seen != 0 {
+		n += 1 + sovDatabase(uint64(m.Seen))
+	}
+	return n
+}
+
+func (m *DatabaseAddress) Size() (n int) {
+	var l int
+	_ = l
+	l = len(m.Address)
+	if l > 0 {
+		n += 1 + l + sovDatabase(uint64(l))
+	}
+	if m.Expires != 0 {
+		n += 1 + sovDatabase(uint64(m.Expires))
+	}
+	return n
+}
+
+func sovDatabase(x uint64) (n int) {
+	for {
+		n++
+		x >>= 7
+		if x == 0 {
+			break
+		}
+	}
+	return n
+}
+func sozDatabase(x uint64) (n int) {
+	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: DatabaseRecord: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: DatabaseRecord: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Addresses = append(m.Addresses, DatabaseAddress{})
+			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Misses", wireType)
+			}
+			m.Misses = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Misses |= (int32(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
+			}
+			m.Seen = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Seen |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Missed", wireType)
+			}
+			m.Missed = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Missed |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ReplicationRecord: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ReplicationRecord: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Key = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Addresses = append(m.Addresses, DatabaseAddress{})
+			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
+			}
+			m.Seen = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Seen |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: DatabaseAddress: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: DatabaseAddress: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Address", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Address = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Expires", wireType)
+			}
+			m.Expires = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Expires |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipDatabase(dAtA []byte) (n int, err error) {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if dAtA[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+			return iNdEx, nil
+		case 1:
+			iNdEx += 8
+			return iNdEx, nil
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			iNdEx += length
+			if length < 0 {
+				return 0, ErrInvalidLengthDatabase
+			}
+			return iNdEx, nil
+		case 3:
+			for {
+				var innerWire uint64
+				var start int = iNdEx
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return 0, ErrIntOverflowDatabase
+					}
+					if iNdEx >= l {
+						return 0, io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					innerWire |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				innerWireType := int(innerWire & 0x7)
+				if innerWireType == 4 {
+					break
+				}
+				next, err := skipDatabase(dAtA[start:])
+				if err != nil {
+					return 0, err
+				}
+				iNdEx = start + next
+			}
+			return iNdEx, nil
+		case 4:
+			return iNdEx, nil
+		case 5:
+			iNdEx += 4
+			return iNdEx, nil
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+	}
+	panic("unreachable")
+}
+
+var (
+	ErrInvalidLengthDatabase = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowDatabase   = fmt.Errorf("proto: integer overflow")
+)
+
+func init() { proto.RegisterFile("database.proto", fileDescriptorDatabase) }
+
+var fileDescriptorDatabase = []byte{
+	// 264 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4b, 0x49, 0x2c, 0x49,
+	0x4c, 0x4a, 0x2c, 0x4e, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0xc9, 0x4d, 0xcc, 0xcc,
+	0x93, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf, 0x4f,
+	0xcf, 0xd7, 0x07, 0x4b, 0x26, 0x95, 0xa6, 0x81, 0x79, 0x60, 0x0e, 0x98, 0x05, 0xd1, 0xa4, 0xd4,
+	0xcf, 0xc8, 0xc5, 0xe7, 0x02, 0x35, 0x27, 0x28, 0x35, 0x39, 0xbf, 0x28, 0x45, 0xc8, 0x92, 0x8b,
+	0x33, 0x31, 0x25, 0xa5, 0x28, 0xb5, 0xb8, 0x38, 0xb5, 0x58, 0x82, 0x51, 0x81, 0x59, 0x83, 0xdb,
+	0x48, 0x54, 0x0f, 0x64, 0xb6, 0x1e, 0x4c, 0xa1, 0x23, 0x44, 0xda, 0x89, 0xe5, 0xc4, 0x3d, 0x79,
+	0x86, 0x20, 0x84, 0x6a, 0x21, 0x31, 0x2e, 0xb6, 0xdc, 0x4c, 0xb0, 0x3e, 0x26, 0x05, 0x46, 0x0d,
+	0xd6, 0x20, 0x28, 0x4f, 0x48, 0x88, 0x8b, 0xa5, 0x38, 0x35, 0x35, 0x4f, 0x82, 0x59, 0x81, 0x51,
+	0x83, 0x39, 0x08, 0xcc, 0x86, 0xab, 0x4d, 0x91, 0x60, 0x01, 0x8b, 0x42, 0x79, 0x4a, 0x25, 0x5c,
+	0x82, 0x41, 0xa9, 0x05, 0x39, 0x99, 0xc9, 0x89, 0x25, 0x99, 0xf9, 0x79, 0x50, 0x37, 0x09, 0x70,
+	0x31, 0x67, 0xa7, 0x56, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0x70, 0x06, 0x81, 0x98, 0xa8, 0xae, 0x64,
+	0x22, 0xc9, 0x95, 0x58, 0x5c, 0xa3, 0xe4, 0xca, 0xc5, 0x8f, 0xa6, 0x4f, 0x48, 0x82, 0x8b, 0x1d,
+	0xaa, 0x07, 0x6a, 0x2f, 0x8c, 0x0b, 0x92, 0x49, 0xad, 0x28, 0xc8, 0x2c, 0x82, 0xfa, 0x93, 0x39,
+	0x08, 0xc6, 0x75, 0x12, 0x38, 0xf1, 0x50, 0x8e, 0xe1, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x8f, 0xe4,
+	0x18, 0x1f, 0x3c, 0x92, 0x63, 0x4c, 0x62, 0x03, 0x87, 0xb3, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff,
+	0x6a, 0x22, 0xa2, 0x85, 0xae, 0x01, 0x00, 0x00,
+}

cmd/stdiscosrv/database.proto

@@ -0,0 +1,33 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+syntax = "proto3";
+
+package main;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_getters_all) = false;
+
+message DatabaseRecord {
+    repeated DatabaseAddress addresses = 1 [(gogoproto.nullable) = false];
+    int32                    misses    = 2; // Number of lookups* without hits
+    int64                    seen      = 3; // Unix nanos, last device announce
+    int64                    missed    = 4; // Unix nanos, last* failed lookup
+}
+
+// *) Not every lookup results in a write, so may not be completely accurate
+
+message ReplicationRecord {
+    string                   key       = 1;
+    repeated DatabaseAddress addresses = 2 [(gogoproto.nullable) = false];
+    int64                    seen      = 3; // Unix nanos, last device announce
+}
+
+message DatabaseAddress {
+    string address = 1;
+    int64  expires = 2; // Unix nanos
+}

cmd/stdiscosrv/database_test.go

@@ -0,0 +1,211 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestDatabaseGetSet(t *testing.T) {
+	os.RemoveAll("_database")
+	defer os.RemoveAll("_database")
+	db, err := newLevelDBStore("_database")
+	if err != nil {
+		t.Fatal(err)
+	}
+	go db.Serve()
+	defer db.Stop()
+
+	// Check missing record
+
+	rec, err := db.get("abcd")
+	if err != nil {
+		t.Error("not found should not be an error")
+	}
+	if len(rec.Addresses) != 0 {
+		t.Error("addresses should be empty")
+	}
+	if rec.Misses != 0 {
+		t.Error("missing should be zero")
+	}
+
+	// Set up a clock
+
+	now := time.Now()
+	tc := &testClock{now}
+	db.clock = tc
+
+	// Put a record
+
+	rec.Addresses = []DatabaseAddress{
+		{Address: "tcp://1.2.3.4:5", Expires: tc.Now().Add(time.Minute).UnixNano()},
+	}
+	if err := db.put("abcd", rec); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it
+
+	rec, err = db.get("abcd")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 1 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have one address")
+	}
+	if rec.Addresses[0].Address != "tcp://1.2.3.4:5" {
+		t.Log(rec.Addresses)
+		t.Error("incorrect address")
+	}
+
+	// Wind the clock one half expiry, and merge in a new address
+
+	tc.wind(30 * time.Second)
+
+	addrs := []DatabaseAddress{
+		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
+	}
+	if err := db.merge("abcd", addrs, tc.Now().UnixNano()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it
+
+	rec, err = db.get("abcd")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 2 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have two addresses")
+	}
+	if rec.Addresses[0].Address != "tcp://1.2.3.4:5" {
+		t.Log(rec.Addresses)
+		t.Error("incorrect address[0]")
+	}
+	if rec.Addresses[1].Address != "tcp://6.7.8.9:0" {
+		t.Log(rec.Addresses)
+		t.Error("incorrect address[1]")
+	}
+
+	// Pass the first expiry time
+
+	tc.wind(45 * time.Second)
+
+	// Verify it
+
+	rec, err = db.get("abcd")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 1 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have one address")
+	}
+	if rec.Addresses[0].Address != "tcp://6.7.8.9:0" {
+		t.Log(rec.Addresses)
+		t.Error("incorrect address")
+	}
+
+	// Put a record with misses
+
+	rec = DatabaseRecord{Misses: 42}
+	if err := db.put("efgh", rec); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it
+
+	rec, err = db.get("efgh")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 0 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have no addresses")
+	}
+	if rec.Misses != 42 {
+		t.Log(rec.Misses)
+		t.Error("incorrect misses")
+	}
+
+	// Set an address
+
+	addrs = []DatabaseAddress{
+		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
+	}
+	if err := db.merge("efgh", addrs, tc.Now().UnixNano()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it
+
+	rec, err = db.get("efgh")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 1 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have one address")
+	}
+	if rec.Misses != 0 {
+		t.Log(rec.Misses)
+		t.Error("should have no misses")
+	}
+}
+
+func TestFilter(t *testing.T) {
+	// all cases are expired with t=10
+	cases := []struct {
+		a []DatabaseAddress
+		b []DatabaseAddress
+	}{
+		{
+			a: nil,
+			b: nil,
+		},
+		{
+			a: []DatabaseAddress{{Address: "a", Expires: 9}, {Address: "b", Expires: 9}, {Address: "c", Expires: 9}},
+			b: []DatabaseAddress{},
+		},
+		{
+			a: []DatabaseAddress{{Address: "a", Expires: 10}},
+			b: []DatabaseAddress{{Address: "a", Expires: 10}},
+		},
+		{
+			a: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
+			b: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
+		},
+		{
+			a: []DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
+			b: []DatabaseAddress{{Address: "d", Expires: 15}, {Address: "b", Expires: 15}}, // gets reordered
+		},
+	}
+
+	for _, tc := range cases {
+		res := expire(tc.a, 10)
+		if fmt.Sprint(res) != fmt.Sprint(tc.b) {
+			t.Errorf("Incorrect result %v, expected %v", res, tc.b)
+		}
+	}
+}
+
+type testClock struct {
+	now time.Time
+}
+
+func (t *testClock) wind(d time.Duration) {
+	t.now = t.now.Add(d)
+}
+
+func (t *testClock) Now() time.Time {
+	return t.now
+}

cmd/stdiscosrv/db.go

@@ -1,32 +0,0 @@
-// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-
-package main
-
-import (
-	"database/sql"
-	"fmt"
-)
-
-type setupFunc func(db *sql.DB) error
-type compileFunc func(db *sql.DB) (map[string]*sql.Stmt, error)
-
-var (
-	setupFuncs   = make(map[string]setupFunc)
-	compileFuncs = make(map[string]compileFunc)
-)
-
-func register(name string, setup setupFunc, compile compileFunc) {
-	setupFuncs[name] = setup
-	compileFuncs[name] = compile
-}
-
-func setup(backend string, db *sql.DB) (map[string]*sql.Stmt, error) {
-	setup, ok := setupFuncs[backend]
-	if !ok {
-		return nil, fmt.Errorf("Unsupported backend")
-	}
-	if err := setup(db); err != nil {
-		return nil, err
-	}
-	return compileFuncs[backend](db)
-}

cmd/stdiscosrv/main.go

@@ -1,29 +1,70 @@
-// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
 	"crypto/tls"
-	"database/sql"
 	"flag"
 	"fmt"
 	"log"
+	"net"
+	"net/http"
 	"os"
 	"runtime"
 	"strconv"
+	"strings"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/thejerf/suture"
 )
 
 const (
-	minNegCache  = 60        // seconds
-	maxNegCache  = 3600      // seconds
-	maxDeviceAge = 7 * 86400 // one week, in seconds
+	addressExpiryTime          = 2 * time.Hour
+	databaseStatisticsInterval = 5 * time.Minute
+
+	// Reannounce-After is set to reannounceAfterSeconds +
+	// random(reannounzeFuzzSeconds), similar for Retry-After
+	reannounceAfterSeconds = 3300
+	reannounzeFuzzSeconds  = 300
+	errorRetryAfterSeconds = 1500
+	errorRetryFuzzSeconds  = 300
+
+	// Retry for not found is minSeconds + failures * incSeconds +
+	// random(fuzz), where failures is the number of consecutive lookups
+	// with no answer, up to maxSeconds. The fuzz is applied after capping
+	// to maxSeconds.
+	notFoundRetryMinSeconds  = 60
+	notFoundRetryMaxSeconds  = 3540
+	notFoundRetryIncSeconds  = 10
+	notFoundRetryFuzzSeconds = 60
+
+	// How often (in requests) we serialize the missed counter to database.
+	notFoundMissesWriteInterval = 10
+
+	httpReadTimeout    = 5 * time.Second
+	httpWriteTimeout   = 5 * time.Second
+	httpMaxHeaderBytes = 1 << 10
+
+	// Size of the replication outbox channel
+	replicationOutboxSize = 10000
 )
 
+// These options make the database a little more optimized for writes, at
+// the expense of some memory usage and risk of losing writes in a (system)
+// crash.
+var levelDBOptions = &opt.Options{
+	NoSync:      true,
+	WriteBuffer: 32 << 20, // default 4<<20
+}
+
 var (
 	Version    string
 	BuildStamp string
@@ -43,17 +84,7 @@ func init() {
 }
 
 var (
-	lruSize     = 10240
-	limitAvg    = 5
-	limitBurst  = 20
-	globalStats stats
-	statsFile   string
-	backend     = "ql"
-	dsn         = getEnvDefault("STDISCOSRV_DB_DSN", "memory://stdiscosrv")
-	certFile    = "cert.pem"
-	keyFile     = "key.pem"
-	debug       = false
-	useHTTP     = false
+	debug = false
 )
 
 func main() {
@@ -63,84 +94,112 @@ func main() {
 	)
 
 	var listen string
+	var dir string
+	var metricsListen string
+	var replicationListen string
+	var replicationPeers string
+	var certFile string
+	var keyFile string
+	var useHTTP bool
 
 	log.SetOutput(os.Stdout)
 	log.SetFlags(0)
 
+	flag.StringVar(&certFile, "cert", "./cert.pem", "Certificate file")
+	flag.StringVar(&dir, "db-dir", "./discovery.db", "Database directory")
+	flag.BoolVar(&debug, "debug", false, "Print debug output")
+	flag.BoolVar(&useHTTP, "http", false, "Listen on HTTP (behind an HTTPS proxy)")
 	flag.StringVar(&listen, "listen", ":8443", "Listen address")
-	flag.IntVar(&lruSize, "limit-cache", lruSize, "Limiter cache entries")
-	flag.IntVar(&limitAvg, "limit-avg", limitAvg, "Allowed average package rate, per 10 s")
-	flag.IntVar(&limitBurst, "limit-burst", limitBurst, "Allowed burst size, packets")
-	flag.StringVar(&statsFile, "stats-file", statsFile, "File to write periodic operation stats to")
-	flag.StringVar(&backend, "db-backend", backend, "Database backend to use")
-	flag.StringVar(&dsn, "db-dsn", dsn, "Database DSN")
-	flag.StringVar(&certFile, "cert", certFile, "Certificate file")
-	flag.StringVar(&keyFile, "key", keyFile, "Key file")
-	flag.BoolVar(&debug, "debug", debug, "Debug")
-	flag.BoolVar(&useHTTP, "http", useHTTP, "Listen on HTTP (behind an HTTPS proxy)")
+	flag.StringVar(&keyFile, "key", "./key.pem", "Key file")
+	flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
+	flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
+	flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
 	flag.Parse()
 
 	log.Println(LongVersion)
 
-	var cert tls.Certificate
-	var err error
-	if !useHTTP {
-		cert, err = tls.LoadX509KeyPair(certFile, keyFile)
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		log.Println("Failed to load keypair. Generating one, this might take a while...")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 0)
 		if err != nil {
-			log.Println("Failed to load keypair. Generating one, this might take a while...")
-			cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 3072)
-			if err != nil {
-				log.Fatalln("Failed to generate X509 key pair:", err)
-			}
+			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
-
-		devID := protocol.NewDeviceID(cert.Certificate[0])
-		log.Println("Server device ID is", devID)
 	}
 
-	db, err := sql.Open(backend, dsn)
-	if err != nil {
-		log.Fatalln("sql.Open:", err)
-	}
-	prep, err := setup(backend, db)
-	if err != nil {
-		log.Fatalln("Setup:", err)
+	devID := protocol.NewDeviceID(cert.Certificate[0])
+	log.Println("Server device ID is", devID)
+
+	// Parse the replication specs, if any.
+	var allowedReplicationPeers []protocol.DeviceID
+	var replicationDestinations []string
+	parts := strings.Split(replicationPeers, ",")
+	for _, part := range parts {
+		fields := strings.Split(part, "@")
+
+		switch len(fields) {
+		case 2:
+			// This is an id@address specification. Grab the address for the
+			// destination list. Try to resolve it once to catch obvious
+			// syntax errors here rather than having the sender service fail
+			// repeatedly later.
+			_, err := net.ResolveTCPAddr("tcp", fields[1])
+			if err != nil {
+				log.Fatalln("Resolving address:", err)
+			}
+			replicationDestinations = append(replicationDestinations, fields[1])
+			fallthrough // N.B.
+
+		case 1:
+			// The first part is always a device ID.
+			id, err := protocol.DeviceIDFromString(fields[0])
+			if err != nil {
+				log.Fatalln("Parsing device ID:", err)
+			}
+			allowedReplicationPeers = append(allowedReplicationPeers, id)
+
+		default:
+			log.Fatalln("Unrecognized replication spec:", part)
+		}
 	}
 
+	// Root of the service tree.
 	main := suture.NewSimple("main")
 
-	main.Add(&querysrv{
-		addr: listen,
-		cert: cert,
-		db:   db,
-		prep: prep,
-	})
+	// Start the database.
+	db, err := newLevelDBStore(dir)
+	if err != nil {
+		log.Fatalln("Open database:", err)
+	}
+	main.Add(db)
 
-	main.Add(&cleansrv{
-		intv: cleanIntv,
-		db:   db,
-		prep: prep,
-	})
+	// Start any replication senders.
+	var repl replicationMultiplexer
+	for _, dst := range replicationDestinations {
+		rs := newReplicationSender(dst, cert, allowedReplicationPeers)
+		main.Add(rs)
+		repl = append(repl, rs)
+	}
 
-	main.Add(&statssrv{
-		intv: statsIntv,
-		file: statsFile,
-		db:   db,
-	})
+	// If we have replication configured, start the replication listener.
+	if len(allowedReplicationPeers) > 0 {
+		rl := newReplicationListener(replicationListen, cert, allowedReplicationPeers, db)
+		main.Add(rl)
+	}
 
-	globalStats.Reset()
+	// Start the main API server.
+	qs := newAPISrv(listen, cert, db, repl, useHTTP)
+	main.Add(qs)
+
+	// If we have a metrics port configured, start a metrics handler.
+	if metricsListen != "" {
+		go func() {
+			mux := http.NewServeMux()
+			mux.Handle("/metrics", promhttp.Handler())
+			log.Fatal(http.ListenAndServe(metricsListen, mux))
+		}()
+	}
+
+	// Engage!
 	main.Serve()
 }
-
-func getEnvDefault(key, def string) string {
-	if val := os.Getenv(key); val != "" {
-		return val
-	}
-	return def
-}
-
-func next(intv time.Duration) time.Duration {
-	t0 := time.Now()
-	t1 := t0.Add(intv).Truncate(intv)
-	return t1.Sub(t0)
-}

cmd/stdiscosrv/psql.go

@@ -1,98 +0,0 @@
-// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-
-package main
-
-import (
-	"database/sql"
-	"fmt"
-
-	_ "github.com/lib/pq"
-)
-
-func init() {
-	register("postgres", postgresSetup, postgresCompile)
-}
-
-func postgresSetup(db *sql.DB) error {
-	var err error
-
-	db.SetMaxIdleConns(4)
-	db.SetMaxOpenConns(8)
-
-	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Devices (
-		DeviceID CHAR(63) NOT NULL PRIMARY KEY,
-		Seen TIMESTAMP NOT NULL
-	)`)
-	if err != nil {
-		return err
-	}
-
-	var tmp string
-	row := db.QueryRow(`SELECT 'DevicesDeviceIDIndex'::regclass`)
-	if err = row.Scan(&tmp); err != nil {
-		_, err = db.Exec(`CREATE INDEX DevicesDeviceIDIndex ON Devices (DeviceID)`)
-	}
-	if err != nil {
-		return err
-	}
-
-	row = db.QueryRow(`SELECT 'DevicesSeenIndex'::regclass`)
-	if err = row.Scan(&tmp); err != nil {
-		_, err = db.Exec(`CREATE INDEX DevicesSeenIndex ON Devices (Seen)`)
-	}
-	if err != nil {
-		return err
-	}
-
-	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Addresses (
-		DeviceID CHAR(63) NOT NULL,
-		Seen TIMESTAMP NOT NULL,
-		Address VARCHAR(2048) NOT NULL
-	)`)
-	if err != nil {
-		return err
-	}
-
-	row = db.QueryRow(`SELECT 'AddressesDeviceIDSeenIndex'::regclass`)
-	if err = row.Scan(&tmp); err != nil {
-		_, err = db.Exec(`CREATE INDEX AddressesDeviceIDSeenIndex ON Addresses (DeviceID, Seen)`)
-	}
-	if err != nil {
-		return err
-	}
-
-	row = db.QueryRow(`SELECT 'AddressesDeviceIDAddressIndex'::regclass`)
-	if err = row.Scan(&tmp); err != nil {
-		_, err = db.Exec(`CREATE INDEX AddressesDeviceIDAddressIndex ON Addresses (DeviceID, Address)`)
-	}
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func postgresCompile(db *sql.DB) (map[string]*sql.Stmt, error) {
-	stmts := map[string]string{
-		"cleanAddress":  "DELETE FROM Addresses WHERE Seen < now() - '2 hour'::INTERVAL",
-		"cleanDevice":   fmt.Sprintf("DELETE FROM Devices WHERE Seen < now() - '%d hour'::INTERVAL", maxDeviceAge/3600),
-		"countAddress":  "SELECT count(*) FROM Addresses",
-		"countDevice":   "SELECT count(*) FROM Devices",
-		"insertAddress": "INSERT INTO Addresses (DeviceID, Seen, Address) VALUES ($1, now(), $2)",
-		"insertDevice":  "INSERT INTO Devices (DeviceID, Seen) VALUES ($1, now())",
-		"selectAddress": "SELECT Address FROM Addresses WHERE DeviceID=$1 AND Seen > now() - '1 hour'::INTERVAL ORDER BY random() LIMIT 16",
-		"selectDevice":  "SELECT Seen FROM Devices WHERE DeviceID=$1",
-		"updateAddress": "UPDATE Addresses SET Seen=now() WHERE DeviceID=$1 AND Address=$2",
-		"updateDevice":  "UPDATE Devices SET Seen=now() WHERE DeviceID=$1",
-	}
-
-	res := make(map[string]*sql.Stmt, len(stmts))
-	for key, stmt := range stmts {
-		prep, err := db.Prepare(stmt)
-		if err != nil {
-			return nil, err
-		}
-		res[key] = prep
-	}
-	return res, nil
-}

cmd/stdiscosrv/ql.go

@@ -1,81 +0,0 @@
-// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
-
-package main
-
-import (
-	"database/sql"
-	"fmt"
-	"log"
-
-	"github.com/cznic/ql"
-)
-
-func init() {
-	ql.RegisterDriver()
-	register("ql", qlSetup, qlCompile)
-}
-
-func qlSetup(db *sql.DB) (err error) {
-	tx, err := db.Begin()
-	if err != nil {
-		return
-	}
-
-	defer func() {
-		if err == nil {
-			err = tx.Commit()
-		} else {
-			tx.Rollback()
-		}
-	}()
-
-	_, err = tx.Exec(`CREATE TABLE IF NOT EXISTS Devices (
-		DeviceID STRING NOT NULL,
-		Seen TIME NOT NULL
-	)`)
-	if err != nil {
-		return
-	}
-
-	if _, err = tx.Exec(`CREATE INDEX IF NOT EXISTS DevicesDeviceIDIndex ON Devices (DeviceID)`); err != nil {
-		return
-	}
-
-	_, err = tx.Exec(`CREATE TABLE IF NOT EXISTS Addresses (
-		DeviceID STRING NOT NULL,
-		Seen TIME NOT NULL,
-		Address STRING NOT NULL,
-	)`)
-	if err != nil {
-		return
-	}
-
-	_, err = tx.Exec(`CREATE INDEX IF NOT EXISTS AddressesDeviceIDAddressIndex ON Addresses (DeviceID, Address)`)
-	return
-}
-
-func qlCompile(db *sql.DB) (map[string]*sql.Stmt, error) {
-	stmts := map[string]string{
-		"cleanAddress":  `DELETE FROM Addresses WHERE Seen < now() - duration("2h")`,
-		"cleanDevice":   fmt.Sprintf(`DELETE FROM Devices WHERE Seen < now() - duration("%dh")`, maxDeviceAge/3600),
-		"countAddress":  "SELECT count(*) FROM Addresses",
-		"countDevice":   "SELECT count(*) FROM Devices",
-		"insertAddress": "INSERT INTO Addresses (DeviceID, Seen, Address) VALUES ($1, now(), $2)",
-		"insertDevice":  "INSERT INTO Devices (DeviceID, Seen) VALUES ($1, now())",
-		"selectAddress": `SELECT Address from Addresses WHERE DeviceID==$1 AND Seen > now() - duration("1h") LIMIT 16`,
-		"selectDevice":  "SELECT Seen FROM Devices WHERE DeviceID==$1",
-		"updateAddress": "UPDATE Addresses Seen=now() WHERE DeviceID==$1 AND Address==$2",
-		"updateDevice":  "UPDATE Devices Seen=now() WHERE DeviceID==$1",
-	}
-
-	res := make(map[string]*sql.Stmt, len(stmts))
-	for key, stmt := range stmts {
-		prep, err := db.Prepare(stmt)
-		if err != nil {
-			log.Println("Failed to compile", stmt)
-			return nil, err
-		}
-		res[key] = prep
-	}
-	return res, nil
-}

cmd/stdiscosrv/querysrv.go

@@ -1,492 +0,0 @@
-// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-
-package main
-
-import (
-	"bytes"
-	"crypto/tls"
-	"database/sql"
-	"encoding/json"
-	"encoding/pem"
-	"fmt"
-	"log"
-	"math/rand"
-	"net"
-	"net/http"
-	"net/url"
-	"strconv"
-	"sync"
-	"time"
-
-	"github.com/golang/groupcache/lru"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"golang.org/x/net/context"
-	"golang.org/x/time/rate"
-)
-
-type querysrv struct {
-	addr     string
-	db       *sql.DB
-	prep     map[string]*sql.Stmt
-	limiter  *safeCache
-	cert     tls.Certificate
-	listener net.Listener
-}
-
-type announcement struct {
-	Seen      time.Time `json:"seen"`
-	Addresses []string  `json:"addresses"`
-}
-
-type safeCache struct {
-	*lru.Cache
-	mut sync.Mutex
-}
-
-func (s *safeCache) Get(key string) (val interface{}, ok bool) {
-	s.mut.Lock()
-	val, ok = s.Cache.Get(key)
-	s.mut.Unlock()
-	return
-}
-
-func (s *safeCache) Add(key string, val interface{}) {
-	s.mut.Lock()
-	s.Cache.Add(key, val)
-	s.mut.Unlock()
-}
-
-type requestID int64
-
-func (i requestID) String() string {
-	return fmt.Sprintf("%016x", int64(i))
-}
-
-type contextKey int
-
-const idKey contextKey = iota
-
-func negCacheFor(lastSeen time.Time) int {
-	since := time.Since(lastSeen).Seconds()
-	if since >= maxDeviceAge {
-		return maxNegCache
-	}
-	if since < 0 {
-		// That's weird
-		return minNegCache
-	}
-
-	// Return a value linearly scaled from minNegCache (at zero seconds ago)
-	// to maxNegCache (at maxDeviceAge seconds ago).
-	r := since / maxDeviceAge
-	return int(minNegCache + r*(maxNegCache-minNegCache))
-}
-
-func (s *querysrv) Serve() {
-	s.limiter = &safeCache{
-		Cache: lru.New(lruSize),
-	}
-
-	if useHTTP {
-		listener, err := net.Listen("tcp", s.addr)
-		if err != nil {
-			log.Println("Listen:", err)
-			return
-		}
-		s.listener = listener
-	} else {
-		tlsCfg := &tls.Config{
-			Certificates:           []tls.Certificate{s.cert},
-			ClientAuth:             tls.RequestClientCert,
-			SessionTicketsDisabled: true,
-			MinVersion:             tls.VersionTLS12,
-			CipherSuites: []uint16{
-				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
-				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
-				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
-				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
-				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
-			},
-		}
-
-		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
-		if err != nil {
-			log.Println("Listen:", err)
-			return
-		}
-		s.listener = tlsListener
-	}
-
-	http.HandleFunc("/v2/", s.handler)
-	http.HandleFunc("/ping", handlePing)
-
-	srv := &http.Server{
-		ReadTimeout:    5 * time.Second,
-		WriteTimeout:   5 * time.Second,
-		MaxHeaderBytes: 1 << 10,
-	}
-
-	if err := srv.Serve(s.listener); err != nil {
-		log.Println("Serve:", err)
-	}
-}
-
-var topCtx = context.Background()
-
-func (s *querysrv) handler(w http.ResponseWriter, req *http.Request) {
-	reqID := requestID(rand.Int63())
-	ctx := context.WithValue(topCtx, idKey, reqID)
-
-	if debug {
-		log.Println(reqID, req.Method, req.URL)
-	}
-
-	t0 := time.Now()
-	defer func() {
-		diff := time.Since(t0)
-		var comment string
-		if diff > time.Second {
-			comment = "(very slow request)"
-		} else if diff > 100*time.Millisecond {
-			comment = "(slow request)"
-		}
-		if comment != "" || debug {
-			log.Println(reqID, req.Method, req.URL, "completed in", diff, comment)
-		}
-	}()
-
-	var remoteIP net.IP
-	if useHTTP {
-		remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
-	} else {
-		addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
-		if err != nil {
-			log.Println("remoteAddr:", err)
-			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-			return
-		}
-		remoteIP = addr.IP
-	}
-
-	if s.limit(remoteIP) {
-		if debug {
-			log.Println(remoteIP, "is limited")
-		}
-		w.Header().Set("Retry-After", "60")
-		http.Error(w, "Too Many Requests", 429)
-		return
-	}
-
-	switch req.Method {
-	case "GET":
-		s.handleGET(ctx, w, req)
-	case "POST":
-		s.handlePOST(ctx, remoteIP, w, req)
-	default:
-		globalStats.Error()
-		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
-	}
-}
-
-func (s *querysrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
-	reqID := ctx.Value(idKey).(requestID)
-
-	deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
-	if err != nil {
-		if debug {
-			log.Println(reqID, "bad device param")
-		}
-		globalStats.Error()
-		http.Error(w, "Bad Request", http.StatusBadRequest)
-		return
-	}
-
-	var ann announcement
-
-	ann.Seen, err = s.getDeviceSeen(deviceID)
-	negCache := strconv.Itoa(negCacheFor(ann.Seen))
-	w.Header().Set("Retry-After", negCache)
-	w.Header().Set("Cache-Control", "public, max-age="+negCache)
-
-	if err != nil {
-		// The device is not in the database.
-		globalStats.Query()
-		http.Error(w, "Not Found", http.StatusNotFound)
-		return
-	}
-
-	t0 := time.Now()
-	ann.Addresses, err = s.getAddresses(ctx, deviceID)
-	if err != nil {
-		log.Println(reqID, "getAddresses:", err)
-		globalStats.Error()
-		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-		return
-	}
-	if debug {
-		log.Println(reqID, "getAddresses in", time.Since(t0))
-	}
-
-	globalStats.Query()
-
-	if len(ann.Addresses) == 0 {
-		http.Error(w, "Not Found", http.StatusNotFound)
-		return
-	}
-
-	globalStats.Answer()
-
-	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(ann)
-}
-
-func (s *querysrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
-	reqID := ctx.Value(idKey).(requestID)
-
-	rawCert := certificateBytes(req)
-	if rawCert == nil {
-		if debug {
-			log.Println(reqID, "no certificates")
-		}
-		globalStats.Error()
-		http.Error(w, "Forbidden", http.StatusForbidden)
-		return
-	}
-
-	var ann announcement
-	if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
-		if debug {
-			log.Println(reqID, "decode:", err)
-		}
-		globalStats.Error()
-		http.Error(w, "Bad Request", http.StatusBadRequest)
-		return
-	}
-
-	deviceID := protocol.NewDeviceID(rawCert)
-
-	// handleAnnounce returns *two* errors. The first indicates a problem with
-	// something the client posted to us. We should return a 400 Bad Request
-	// and not worry about it. The second indicates that the request was fine,
-	// but something internal messed up. We should log it and respond with a
-	// more apologetic 500 Internal Server Error.
-	userErr, internalErr := s.handleAnnounce(ctx, remoteIP, deviceID, ann.Addresses)
-	if userErr != nil {
-		if debug {
-			log.Println(reqID, "handleAnnounce:", userErr)
-		}
-		globalStats.Error()
-		http.Error(w, "Bad Request", http.StatusBadRequest)
-		return
-	}
-	if internalErr != nil {
-		log.Println(reqID, "handleAnnounce:", internalErr)
-		globalStats.Error()
-		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-		return
-	}
-
-	globalStats.Announce()
-
-	// TODO: Slowly increase this for stable clients
-	w.Header().Set("Reannounce-After", "1800")
-
-	// We could return the lookup result here, but it's kind of unnecessarily
-	// expensive to go query the database again so we let the client decide to
-	// do a lookup if they really care.
-	w.WriteHeader(http.StatusNoContent)
-}
-
-func (s *querysrv) Stop() {
-	s.listener.Close()
-}
-
-func (s *querysrv) handleAnnounce(ctx context.Context, remote net.IP, deviceID protocol.DeviceID, addresses []string) (userErr, internalErr error) {
-	reqID := ctx.Value(idKey).(requestID)
-
-	tx, err := s.db.Begin()
-	if err != nil {
-		internalErr = err
-		return
-	}
-
-	defer func() {
-		// Since we return from a bunch of different places, we handle
-		// rollback in the defer.
-		if internalErr != nil || userErr != nil {
-			tx.Rollback()
-		}
-	}()
-
-	for _, annAddr := range addresses {
-		uri, err := url.Parse(annAddr)
-		if err != nil {
-			userErr = err
-			return
-		}
-
-		host, port, err := net.SplitHostPort(uri.Host)
-		if err != nil {
-			userErr = err
-			return
-		}
-
-		ip := net.ParseIP(host)
-		if host == "" || ip.IsUnspecified() {
-			// Do not use IPv6 remote address if requested scheme is tcp4
-			if uri.Scheme == "tcp4" && remote.To4() == nil {
-				continue
-			}
-
-			// Do not use IPv4 remote address if requested scheme is tcp6
-			if uri.Scheme == "tcp6" && remote.To4() != nil {
-				continue
-			}
-
-			host = remote.String()
-		}
-
-		uri.Host = net.JoinHostPort(host, port)
-
-		if err := s.updateAddress(ctx, tx, deviceID, uri.String()); err != nil {
-			internalErr = err
-			return
-		}
-	}
-
-	if err := s.updateDevice(ctx, tx, deviceID); err != nil {
-		internalErr = err
-		return
-	}
-
-	t0 := time.Now()
-	internalErr = tx.Commit()
-	if debug {
-		log.Println(reqID, "commit in", time.Since(t0))
-	}
-	return
-}
-
-func (s *querysrv) limit(remote net.IP) bool {
-	key := remote.String()
-
-	bkt, ok := s.limiter.Get(key)
-	if ok {
-		bkt := bkt.(*rate.Limiter)
-		if !bkt.Allow() {
-			// Rate limit exceeded; ignore packet
-			return true
-		}
-	} else {
-		// limitAvg is in packets per ten seconds.
-		s.limiter.Add(key, rate.NewLimiter(rate.Limit(limitAvg)/10, limitBurst))
-	}
-
-	return false
-}
-
-func (s *querysrv) updateDevice(ctx context.Context, tx *sql.Tx, device protocol.DeviceID) error {
-	reqID := ctx.Value(idKey).(requestID)
-	t0 := time.Now()
-	res, err := tx.Stmt(s.prep["updateDevice"]).Exec(device.String())
-	if err != nil {
-		return err
-	}
-	if debug {
-		log.Println(reqID, "updateDevice in", time.Since(t0))
-	}
-
-	if rows, _ := res.RowsAffected(); rows == 0 {
-		t0 = time.Now()
-		_, err := tx.Stmt(s.prep["insertDevice"]).Exec(device.String())
-		if err != nil {
-			return err
-		}
-		if debug {
-			log.Println(reqID, "insertDevice in", time.Since(t0))
-		}
-	}
-
-	return nil
-}
-
-func (s *querysrv) updateAddress(ctx context.Context, tx *sql.Tx, device protocol.DeviceID, uri string) error {
-	res, err := tx.Stmt(s.prep["updateAddress"]).Exec(device.String(), uri)
-	if err != nil {
-		return err
-	}
-
-	if rows, _ := res.RowsAffected(); rows == 0 {
-		_, err := tx.Stmt(s.prep["insertAddress"]).Exec(device.String(), uri)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (s *querysrv) getAddresses(ctx context.Context, device protocol.DeviceID) ([]string, error) {
-	rows, err := s.prep["selectAddress"].Query(device.String())
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	var res []string
-	for rows.Next() {
-		var addr string
-
-		err := rows.Scan(&addr)
-		if err != nil {
-			log.Println("Scan:", err)
-			continue
-		}
-		res = append(res, addr)
-	}
-
-	return res, nil
-}
-
-func (s *querysrv) getDeviceSeen(device protocol.DeviceID) (time.Time, error) {
-	row := s.prep["selectDevice"].QueryRow(device.String())
-	var seen time.Time
-	if err := row.Scan(&seen); err != nil {
-		return time.Time{}, err
-	}
-	return seen.In(time.UTC), nil
-}
-
-func handlePing(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(204)
-}
-
-func certificateBytes(req *http.Request) []byte {
-	if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
-		return req.TLS.PeerCertificates[0].Raw
-	}
-
-	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
-		bs := []byte(hdr)
-		// The certificate is in PEM format but with spaces for newlines. We
-		// need to reinstate the newlines for the PEM decoder. But we need to
-		// leave the spaces in the BEGIN and END lines - the first and last
-		// space - alone.
-		firstSpace := bytes.Index(bs, []byte(" "))
-		lastSpace := bytes.LastIndex(bs, []byte(" "))
-		for i := firstSpace + 1; i < lastSpace; i++ {
-			if bs[i] == ' ' {
-				bs[i] = '\n'
-			}
-		}
-		block, _ := pem.Decode(bs)
-		if block == nil {
-			// Decoding failed
-			return nil
-		}
-		return block.Bytes
-	}
-
-	return nil
-}

cmd/stdiscosrv/replication.go

@@ -0,0 +1,304 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"crypto/tls"
+	"encoding/binary"
+	"fmt"
+	io "io"
+	"log"
+	"net"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+type replicator interface {
+	send(key string, addrs []DatabaseAddress, seen int64)
+}
+
+// a replicationSender tries to connect to the remote address and provide
+// them with a feed of replication updates.
+type replicationSender struct {
+	dst        string
+	cert       tls.Certificate // our certificate
+	allowedIDs []protocol.DeviceID
+	outbox     chan ReplicationRecord
+	stop       chan struct{}
+}
+
+func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
+	return &replicationSender{
+		dst:        dst,
+		cert:       cert,
+		allowedIDs: allowedIDs,
+		outbox:     make(chan ReplicationRecord, replicationOutboxSize),
+		stop:       make(chan struct{}),
+	}
+}
+
+func (s *replicationSender) Serve() {
+	// Sleep a little at startup. Peers often restart at the same time, and
+	// this avoid the service failing and entering backoff state
+	// unnecessarily, while also reducing the reconnect rate to something
+	// reasonable by default.
+	time.Sleep(2 * time.Second)
+
+	tlsCfg := &tls.Config{
+		Certificates:       []tls.Certificate{s.cert},
+		MinVersion:         tls.VersionTLS12,
+		InsecureSkipVerify: true,
+	}
+
+	// Dial the TLS connection.
+	conn, err := tls.Dial("tcp", s.dst, tlsCfg)
+	if err != nil {
+		log.Println("Replication connect:", err)
+		return
+	}
+	defer func() {
+		conn.SetWriteDeadline(time.Now().Add(time.Second))
+		conn.Close()
+	}()
+
+	// Get the other side device ID.
+	remoteID, err := deviceID(conn)
+	if err != nil {
+		log.Println("Replication connect:", err)
+		return
+	}
+
+	// Verify it's in the set of allowed device IDs.
+	if !deviceIDIn(remoteID, s.allowedIDs) {
+		log.Println("Replication connect: unexpected device ID:", remoteID)
+		return
+	}
+
+	// Send records.
+	buf := make([]byte, 1024)
+	for {
+		select {
+		case rec := <-s.outbox:
+			// Buffer must hold record plus four bytes for size
+			size := rec.Size()
+			if len(buf) < size+4 {
+				buf = make([]byte, size+4)
+			}
+
+			// Record comes after the four bytes size
+			n, err := rec.MarshalTo(buf[4:])
+			if err != nil {
+				// odd to get an error here, but we haven't sent anything
+				// yet so it's not fatal
+				replicationSendsTotal.WithLabelValues("error").Inc()
+				log.Println("Replication marshal:", err)
+				continue
+			}
+			binary.BigEndian.PutUint32(buf, uint32(n))
+
+			// Send
+			conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
+			if _, err := conn.Write(buf[:4+n]); err != nil {
+				replicationSendsTotal.WithLabelValues("error").Inc()
+				log.Println("Replication write:", err)
+				return
+			}
+			replicationSendsTotal.WithLabelValues("success").Inc()
+
+		case <-s.stop:
+			return
+		}
+	}
+}
+
+func (s *replicationSender) Stop() {
+	close(s.stop)
+}
+
+func (s *replicationSender) String() string {
+	return fmt.Sprintf("replicationSender(%q)", s.dst)
+}
+
+func (s *replicationSender) send(key string, ps []DatabaseAddress, seen int64) {
+	item := ReplicationRecord{
+		Key:       key,
+		Addresses: ps,
+	}
+
+	// The send should never block. The inbox is suitably buffered for at
+	// least a few seconds of stalls, which shouldn't happen in practice.
+	select {
+	case s.outbox <- item:
+	default:
+		replicationSendsTotal.WithLabelValues("drop").Inc()
+	}
+}
+
+// a replicationMultiplexer sends to multiple replicators
+type replicationMultiplexer []replicator
+
+func (m replicationMultiplexer) send(key string, ps []DatabaseAddress, seen int64) {
+	for _, s := range m {
+		// each send is nonblocking
+		s.send(key, ps, seen)
+	}
+}
+
+// replicationListener accepts incoming connections and reads replication
+// items from them. Incoming items are applied to the KV store.
+type replicationListener struct {
+	addr       string
+	cert       tls.Certificate
+	allowedIDs []protocol.DeviceID
+	db         database
+	stop       chan struct{}
+}
+
+func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
+	return &replicationListener{
+		addr:       addr,
+		cert:       cert,
+		allowedIDs: allowedIDs,
+		db:         db,
+		stop:       make(chan struct{}),
+	}
+}
+
+func (l *replicationListener) Serve() {
+	tlsCfg := &tls.Config{
+		Certificates:       []tls.Certificate{l.cert},
+		ClientAuth:         tls.RequestClientCert,
+		MinVersion:         tls.VersionTLS12,
+		InsecureSkipVerify: true,
+	}
+
+	lst, err := tls.Listen("tcp", l.addr, tlsCfg)
+	if err != nil {
+		log.Println("Replication listen:", err)
+		return
+	}
+	defer lst.Close()
+
+	for {
+		select {
+		case <-l.stop:
+			return
+		default:
+		}
+
+		// Accept a connection
+		conn, err := lst.Accept()
+		if err != nil {
+			log.Println("Replication accept:", err)
+			return
+		}
+
+		// Figure out the other side device ID
+		remoteID, err := deviceID(conn.(*tls.Conn))
+		if err != nil {
+			log.Println("Replication accept:", err)
+			conn.SetWriteDeadline(time.Now().Add(time.Second))
+			conn.Close()
+			continue
+		}
+
+		// Verify it is in the set of allowed device IDs
+		if !deviceIDIn(remoteID, l.allowedIDs) {
+			log.Println("Replication accept: unexpected device ID:", remoteID)
+			conn.SetWriteDeadline(time.Now().Add(time.Second))
+			conn.Close()
+			continue
+		}
+
+		go l.handle(conn)
+	}
+}
+
+func (l *replicationListener) Stop() {
+	close(l.stop)
+}
+
+func (l *replicationListener) String() string {
+	return fmt.Sprintf("replicationListener(%q)", l.addr)
+}
+
+func (l *replicationListener) handle(conn net.Conn) {
+	defer func() {
+		conn.SetWriteDeadline(time.Now().Add(time.Second))
+		conn.Close()
+	}()
+
+	buf := make([]byte, 1024)
+
+	for {
+		select {
+		case <-l.stop:
+			return
+		default:
+		}
+
+		conn.SetReadDeadline(time.Now().Add(time.Minute))
+
+		// First four bytes are the size
+		if _, err := io.ReadFull(conn, buf[:4]); err != nil {
+			log.Println("Replication read size:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			return
+		}
+
+		// Read the rest of the record
+		size := int(binary.BigEndian.Uint32(buf[:4]))
+		if len(buf) < size {
+			buf = make([]byte, size)
+		}
+		if _, err := io.ReadFull(conn, buf[:size]); err != nil {
+			log.Println("Replication read record:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			return
+		}
+
+		// Unmarshal
+		var rec ReplicationRecord
+		if err := rec.Unmarshal(buf[:size]); err != nil {
+			log.Println("Replication unmarshal:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			continue
+		}
+
+		// Store
+		l.db.merge(rec.Key, rec.Addresses, rec.Seen)
+		replicationRecvsTotal.WithLabelValues("success").Inc()
+	}
+}
+
+func deviceID(conn *tls.Conn) (protocol.DeviceID, error) {
+	// Handshake may not be complete on the server side yet, which we need
+	// to get the client certificate.
+	if !conn.ConnectionState().HandshakeComplete {
+		if err := conn.Handshake(); err != nil {
+			return protocol.DeviceID{}, err
+		}
+	}
+
+	// We expect exactly one certificate.
+	certs := conn.ConnectionState().PeerCertificates
+	if len(certs) != 1 {
+		return protocol.DeviceID{}, fmt.Errorf("unexpected number of certificates (%d != 1)", len(certs))
+	}
+
+	return protocol.NewDeviceID(certs[0].Raw), nil
+}
+
+func deviceIDIn(id protocol.DeviceID, ids []protocol.DeviceID) bool {
+	for _, candidate := range ids {
+		if id == candidate {
+			return true
+		}
+	}
+	return false
+}

cmd/stdiscosrv/stats.go

@@ -1,141 +1,112 @@
-// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
-	"bytes"
-	"database/sql"
-	"fmt"
-	"io/ioutil"
-	"log"
 	"os"
-	"sync/atomic"
-	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
 )
 
-type stats struct {
-	// Incremented atomically
-	announces int64
-	queries   int64
-	answers   int64
-	errors    int64
-}
-
-func (s *stats) Announce() {
-	atomic.AddInt64(&s.announces, 1)
-}
-
-func (s *stats) Query() {
-	atomic.AddInt64(&s.queries, 1)
-}
-
-func (s *stats) Answer() {
-	atomic.AddInt64(&s.answers, 1)
-}
-
-func (s *stats) Error() {
-	atomic.AddInt64(&s.errors, 1)
-}
-
-// Reset returns a copy of the current stats and resets the counters to
-// zero.
-func (s *stats) Reset() stats {
-	// Create a copy of the stats using atomic reads
-	copy := stats{
-		announces: atomic.LoadInt64(&s.announces),
-		queries:   atomic.LoadInt64(&s.queries),
-		answers:   atomic.LoadInt64(&s.answers),
-		errors:    atomic.LoadInt64(&s.errors),
-	}
-
-	// Reset the stats by subtracting the values that we copied
-	atomic.AddInt64(&s.announces, -copy.announces)
-	atomic.AddInt64(&s.queries, -copy.queries)
-	atomic.AddInt64(&s.answers, -copy.answers)
-	atomic.AddInt64(&s.errors, -copy.errors)
-
-	return copy
-}
-
-type statssrv struct {
-	intv time.Duration
-	file string
-	db   *sql.DB
-}
-
-func (s *statssrv) Serve() {
-	lastReset := time.Now()
-	for {
-		time.Sleep(next(s.intv))
-
-		stats := globalStats.Reset()
-		d := time.Since(lastReset).Seconds()
-		lastReset = time.Now()
-
-		log.Printf("Stats: %.02f announces/s, %.02f queries/s, %.02f answers/s, %.02f errors/s",
-			float64(stats.announces)/d, float64(stats.queries)/d, float64(stats.answers)/d, float64(stats.errors)/d)
-
-		if s.file != "" {
-			s.writeToFile(stats, d)
-		}
-	}
-}
-
-func (s *statssrv) Stop() {
-	panic("stop unimplemented")
-}
-
-func (s *statssrv) writeToFile(stats stats, secs float64) {
-	newLine := []byte("\n")
-
-	var addrs int
-	row := s.db.QueryRow("SELECT COUNT(*) FROM Addresses")
-	if err := row.Scan(&addrs); err != nil {
-		log.Println("stats query:", err)
-		return
-	}
-
-	fd, err := os.OpenFile(s.file, os.O_RDWR|os.O_CREATE, 0666)
-	if err != nil {
-		log.Println("stats file:", err)
-		return
-	}
-	defer func() {
-		err = fd.Close()
-		if err != nil {
-			log.Println("stats file:", err)
-		}
-	}()
-
-	bs, err := ioutil.ReadAll(fd)
-	if err != nil {
-		log.Println("stats file:", err)
-		return
-	}
-	lines := bytes.Split(bytes.TrimSpace(bs), newLine)
-	if len(lines) > 12 {
-		lines = lines[len(lines)-12:]
-	}
-
-	latest := fmt.Sprintf("%v: %6d addresses, %8.02f announces/s, %8.02f queries/s, %8.02f answers/s, %8.02f errors/s\n",
-		time.Now().UTC().Format(time.RFC3339), addrs,
-		float64(stats.announces)/secs, float64(stats.queries)/secs, float64(stats.answers)/secs, float64(stats.errors)/secs)
-	lines = append(lines, []byte(latest))
-
-	_, err = fd.Seek(0, 0)
-	if err != nil {
-		log.Println("stats file:", err)
-		return
-	}
-	err = fd.Truncate(0)
-	if err != nil {
-		log.Println("stats file:", err)
-		return
-	}
-
-	_, err = fd.Write(bytes.Join(lines, newLine))
-	if err != nil {
-		log.Println("stats file:", err)
-		return
-	}
+var (
+	apiRequestsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "api_requests_total",
+			Help:      "Number of API requests.",
+		}, []string{"type", "result"})
+	apiRequestsSeconds = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Namespace:  "syncthing",
+			Subsystem:  "discovery",
+			Name:       "api_requests_seconds",
+			Help:       "Latency of API requests.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		}, []string{"type"})
+
+	lookupRequestsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "lookup_requests_total",
+			Help:      "Number of lookup requests.",
+		}, []string{"result"})
+	announceRequestsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "announcement_requests_total",
+			Help:      "Number of announcement requests.",
+		}, []string{"result"})
+
+	replicationSendsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "replication_sends_total",
+			Help:      "Number of replication sends.",
+		}, []string{"result"})
+	replicationRecvsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "replication_recvs_total",
+			Help:      "Number of replication receives.",
+		}, []string{"result"})
+
+	databaseKeys = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "database_keys",
+			Help:      "Number of database keys at last count.",
+		}, []string{"category"})
+	databaseStatisticsSeconds = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "database_statistics_seconds",
+			Help:      "Time spent running the statistics routine.",
+		})
+
+	databaseOperations = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "database_operations_total",
+			Help:      "Number of database operations.",
+		}, []string{"operation", "result"})
+	databaseOperationSeconds = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Namespace:  "syncthing",
+			Subsystem:  "discovery",
+			Name:       "database_operation_seconds",
+			Help:       "Latency of database operations.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		}, []string{"operation"})
+)
+
+const (
+	dbOpGet             = "get"
+	dbOpPut             = "put"
+	dbOpMerge           = "merge"
+	dbResSuccess        = "success"
+	dbResNotFound       = "not_found"
+	dbResError          = "error"
+	dbResUnmarshalError = "unmarsh_err"
+)
+
+func init() {
+	prometheus.MustRegister(apiRequestsTotal, apiRequestsSeconds,
+		lookupRequestsTotal, announceRequestsTotal,
+		replicationSendsTotal, replicationRecvsTotal,
+		databaseKeys, databaseStatisticsSeconds,
+		databaseOperations, databaseOperationSeconds)
+
+	prometheus.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "syncthing_discovery"))
 }

cmd/stevents/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/stfileinfo/main.go

@@ -2,11 +2,12 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
+	"context"
 	"flag"
 	"log"
 	"os"
@@ -70,7 +71,7 @@ func main() {
 		if *standardBlocks || blockSize < protocol.BlockSize {
 			blockSize = protocol.BlockSize
 		}
-		bs, err := scanner.Blocks(fd, blockSize, fi.Size(), nil, true)
+		bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/stfinddevice/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/stgenfiles/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/stindex/dump.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/stindex/dumpsize.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/stindex/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/stindex/util.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -12,7 +12,7 @@ import (
 	"path/filepath"
 	"runtime"
 
-	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/fs"
 )
 
 func nulString(bs []byte) string {
@@ -33,7 +33,7 @@ func defaultConfigDir() string {
 		return filepath.Join(os.Getenv("AppData"), "Syncthing")
 
 	case "darwin":
-		dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
+		dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -43,7 +43,7 @@ func defaultConfigDir() string {
 		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
 			return filepath.Join(xdgCfg, "syncthing")
 		}
-		dir, err := osutil.ExpandTilde("~/.config/syncthing")
+		dir, err := fs.ExpandTilde("~/.config/syncthing")
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/strelaypoolsrv/README.md

@@ -1,16 +1,19 @@
 # relaypoolsrv
 
-[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/relaypoolsrv.svg?style=flat-square)](http://build.syncthing.net/job/relaypoolsrv/lastBuild/)
+This is the relay pool server for the `syncthing` project, which allows
+community hosted [relaysrv](https://github.com/syncthing/relaysrv)'s to join
+the public pool.
 
-This is the relay pool server for the `syncthing` project, which allows community hosted [relaysrv](https://github.com/syncthing/relaysrv)'s to join the public pool.
+Servers that join the pool are then advertised to users of `syncthing` as
+potential connection points for those who are unable to connect directly due
+to NAT or firewall issues.
 
-Servers that join the pool are then advertised to users of `syncthing` as potential connection points for those who are unable to connect directly due to NAT or firewall issues.
-
-There is very little reason why you'd want to run this yourself, as `relaypoolsrv` is just used for announcement and lookup of public relay servers. If you are looking to setup a private or a public relay, please check the documentation for [relaysrv](https://github.com/syncthing/relaysrv), which also explains how to join the default public pool.
-
-If you still want to run it, you can run `go get github.com/syncthing/relaypoolsrv` download it or download the
-[latest build](http://build.syncthing.net/job/relaypoolsrv/lastSuccessfulBuild/artifact/)
-from the build server.
+There is very little reason why you'd want to run this yourself, as
+`relaypoolsrv` is just used for announcement and lookup of public relay
+servers. If you are looking to setup a private or a public relay, please
+check the documentation for
+[relaysrv](https://github.com/syncthing/relaysrv), which also explains how
+to join the default public pool.
 
 See `relaypoolsrv -help` for configuration options.
 

cmd/strelaypoolsrv/gui/index.html

@@ -2,15 +2,15 @@
 
 <html lang="en" ng-app="syncthing" ng-controller="relayDataController">
   <head>
-    <meta charset="utf-8">
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <meta name="description" content="">
-    <meta name="author" content="">
+    <meta charset="utf-8"/>
+    <meta http-equiv="X-UA-Compatible" content="IE=edge"/>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+    <meta name="description" content=""/>
+    <meta name="author" content=""/>
 
     <title>Relay stats</title>
-    <link href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet">
-    <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/font-awesome/4.6.1/css/font-awesome.min.css">
+    <link href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
+    <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/font-awesome/4.6.1/css/font-awesome.min.css"/>
 
     <style>
       #map {
@@ -36,9 +36,9 @@
 
   <body class="ng-cloak">
     <div class="container">
-      <h1>Relay Pool Data</h2>
+      <h1>Relay Pool Data</h1>
       <div ng-if="relays === undefined" class="text-center">
-        <img src="//cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif"/>
+        <img src="//cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
         <p>Please wait while we gather data</p>
       </div>
       <div>
@@ -48,7 +48,7 @@
           </p>
         </div>
         <div id="map"></div> <!-- Can't hide the map, otherwise it freaks out -->
-        <p>The circle size represents how much bytes the relay transfered relative to other relays</p>
+        <p>The circle size represents how much bytes the relay transferred relative to other relays</p>
       </div>
       <div>
         <table class="table table-striped table-condensed table">
@@ -56,105 +56,105 @@
             <tr>
               <th rowspan="2">Address</td>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.numActiveSessions'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.numActiveSessions'; sortReverse = !sortReverse">
                   Sessions
-                  <span ng-show="sortType == 'status.numActiveSessions' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.numActiveSessions' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.numActiveSessions' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.numActiveSessions' && sortReverse" class="fa fa-caret-up"></span>
                 </a>
               </th>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.numConnections'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.numConnections'; sortReverse = !sortReverse">
                   Connections
-                  <span ng-show="sortType == 'status.numConnections' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.numConnections' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.numConnections' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.numConnections' && sortReverse" class="fa fa-caret-up"></span>
                 </a>
               </th>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.bytesProxied'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.bytesProxied'; sortReverse = !sortReverse">
                   Data relayed
-                  <span ng-show="sortType == 'status.bytesProxied' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.bytesProxied' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.bytesProxied' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.bytesProxied' && sortReverse" class="fa fa-caret-up"></span>
                 </a>
               </th>
               <th colspan="6" class="text-center">Transfer rate in the last period</th>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.uptimeSeconds'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.uptimeSeconds'; sortReverse = !sortReverse">
                   Uptime hours
-                  <span ng-show="sortType == 'status.uptimeSeconds' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.uptimeSeconds' && !sortReverse" class="fa fa-caret-down"></span>
                   <span ng-show="sortType == 'status.uptimeSeconds' && sortReverse" class="fa fa-caret-up"></span>
                 </a>
               </th>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.options[\'provided-by\'] || \'\''; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.options[\'provided-by\'] || \'\''; sortReverse = !sortReverse">
                   Provided by
-                  <span ng-show="sortType == 'status.options[\'provided-by\'] || \'\'' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.options[\'provided-by\'] || \'\'' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.options[\'provided-by\'] || \'\'' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.options[\'provided-by\'] || \'\'' && sortReverse" class="fa fa-caret-up"></span>
                 </a>
               </th>
             </tr>
             <tr>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[0]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[0]'; sortReverse = !sortReverse">
                   10s
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[0]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[0]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[0]' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[0]' && sortReverse" class="fa fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[1]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[1]'; sortReverse = !sortReverse">
                   1m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[1]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[1]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[1]' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[1]' && sortReverse" class="fa fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[2]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[2]'; sortReverse = !sortReverse">
                   5m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[2]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[2]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[2]' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[2]' && sortReverse" class="fa fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[3]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[3]'; sortReverse = !sortReverse">
                   15m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[3]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[3]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[3]' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[3]' && sortReverse" class="fa fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[4]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[4]'; sortReverse = !sortReverse">
                   30m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[4]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[4]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[4]' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[4]' && sortReverse" class="fa fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[5]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[5]'; sortReverse = !sortReverse">
                   60m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[5]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[5]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[5]' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[5]' && sortReverse" class="fa fa-caret-up"></span>
                 </a>
               </th>
             </tr>
           </thead>
           <tbody>
-            <tr ng-repeat="relay in relays | orderBy:sortType:sortReverse:sortCompare">
+            <tr ng-repeat="relay in relays | orderBy:sortType:sortReverse:sortCompare" ng-mouseover="relay.showMarker()" ng-mouseleave="relay.hideMarker()">
               <td>{{ relay.address }}</td>
-              <td ng-if="relay.status === undefined" colspan="11" class="text-center">Looking up...</td>
-              <td ng-if-start="relay.status !== undefined">{{ relay.status.numActiveSessions }}</td>
-              <td>{{ relay.status.numConnections }}</td>
-              <td>{{ relay.status.bytesProxied | bytes }}</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
-              <td ng-if="relay.status.uptimeSeconds != undefined">{{ relay.status.uptimeSeconds/60/60 | number:0 }}</td>
-              <td ng-if="relay.status.uptimeSeconds == undefined"></td>
-              <td title="{{ relay.status.options['provided-by'] || '' }}" ng-if-end>
-                {{ relay.status.options['provided-by'] || '' | limitTo:50 }}
-                <span ng-if="(relay.status.options['provided-by'] || '').length > 50">&hellip;
+              <td ng-if="!relay.stats" colspan="11"></td>
+              <td ng-if-start="relay.stats">{{ relay.stats.numActiveSessions }}</td>
+              <td>{{ relay.stats.numConnections }}</td>
+              <td>{{ relay.stats.bytesProxied | bytes }}</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
+              <td ng-if="relay.stats.uptimeSeconds != undefined">{{ relay.stats.uptimeSeconds/60/60 | number:0 }}</td>
+              <td ng-if="relay.stats.uptimeSeconds == undefined"></td>
+              <td title="{{ relay.stats.options['provided-by'] || '' }}" ng-if-end>
+                {{ relay.stats.options['provided-by'] || '' | limitTo:50 }}
+                <span ng-if="(relay.stats.options['provided-by'] || '').length > 50">&hellip;
               </td>
             </tr>
           </tbody>
@@ -184,10 +184,10 @@
     </div>
 
 
-    <script src="//code.jquery.com/jquery-2.1.4.min.js"></script>
-    <script src="//cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
-    <script src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
-    <script src="//maps.googleapis.com/maps/api/js"></script>
+    <script type="text/javascript" src="//code.jquery.com/jquery-2.1.4.min.js"></script>
+    <script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
+    <script type="text/javascript" src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
+    <script type="text/javascript" src="//maps.googleapis.com/maps/api/js"></script>
   </body>
 
   <script>
@@ -235,16 +235,16 @@
     $scope.mapBounds = new google.maps.LatLngBounds();
     $scope.tooltipTemplate = $('#infoTemplate').html();
     $scope.usedLocations = {};
-    $scope.sortType = 'status.numActiveSessions';
+    $scope.sortType = 'stats.numActiveSessions';
     $scope.sortReverse = true;
     $scope.sortCompare = function(a, b) {
       if (a.value == b.value) {
         return 0;
       }
-      if (a.type == "undefined") {
+      if (a.type == "undefined" || a.type == "null") {
         return -1;
       }
-      if (b.type == "undefined") {
+      if (b.type == "undefined" || b.type == "null") {
         return 1;
       }
       return a.value > b.value ? 1 : -1;
@@ -252,25 +252,31 @@
 
     $http.get("/endpoint").then(function(response) {
       $scope.relays = response.data.relays;
-      var promises = [];
-      angular.forEach($scope.relays, function(relay) {
 
+      angular.forEach($scope.relays, function(relay) {
         relay.uri = constructURI(relay.url);
         relay.address = relay.url.split('/')[2];
 
         addMarkerToMap(relay);
 
-        promises.push(getRelayStatus(relay));
+        if (relay.stats) {
+          angular.forEach($scope.totals, function(value, key) {
+            if (typeof $scope.totals[key] == 'number') {
+              $scope.totals[key] += relay.stats[key];
+            } else if (typeof $scope.totals[key] == 'object' && $scope.totals[key] instanceof Array) {
+              angular.forEach($scope.totals[key], function(value, index) {
+                $scope.totals[key][index] += relay.stats[key][index];
+              });
+            }
+          });
+        }
       });
 
-      // Can only add circles once we know the totals for transfers, which means
-      // we need to resolve all statuses.
-      $q.all(promises).then(function() {
-        angular.forEach($scope.relays, function(relay) {
-          if (relay.status) {
-            addCircleToMap(relay);
-          }
-        });
+      // After the totals were calculated, add circles.
+      angular.forEach($scope.relays, function(relay) {
+        if (relay.stats) {
+          addCircleToMap(relay);
+        }
       });
 
       $scope.map.fitBounds($scope.mapBounds);
@@ -307,13 +313,16 @@
           content: $compile($scope.tooltipTemplate)(scope)[0],
         });
 
-        relay.marker.addListener('mouseover', function() {
+        relay.showMarker = function() {
           relay.marker.info.open($scope.map, relay.marker);
-        });
+        }
 
-        relay.marker.addListener('mouseout', function() {
+        relay.hideMarker = function() {
           relay.marker.info.close();
-        });
+        }
+
+        relay.marker.addListener('mouseover', relay.showMarker);
+        relay.marker.addListener('mouseout', relay.hideMarker);
 
         $scope.mapBounds.extend(relay.marker.position);
     }
@@ -327,41 +336,10 @@
         fillOpacity: 0.35,
         map: $scope.map,
         center: relay.marker.position,
-        radius: ((relay.status.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000
+        radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000
       });
     }
 
-    function getRelayStatus(relay) {
-      // Normal timeout doesn't deal with relays which accept the TCP connection
-      // but don't respond (some firewalls do that), so deal with it this way.
-      var timeoutRequest = $q.defer();
-      var resolveStatus = $q.defer();
-
-
-      $http.get("http://" + relay.uri.hostname + ':' + ((relay.uri.args.statusAddr && relay.uri.args.statusAddr.split(':')[1]) || "22070") + "/status", { timeout: timeoutRequest.promise }).then(function (response) {
-        relay.status = response.data;
-        resolveStatus.resolve();
-        angular.forEach($scope.totals, function(value, key) {
-          if (typeof $scope.totals[key] == 'number') {
-            $scope.totals[key] += response.data[key];
-          } else if (typeof $scope.totals[key] == 'object' && $scope.totals[key] instanceof Array) {
-            angular.forEach($scope.totals[key], function(value, index) {
-              $scope.totals[key][index] += response.data[key][index];
-            });
-          }
-        });
-      }, function() {
-        relay.status = null;
-        resolveStatus.resolve();
-      });
-
-      $timeout(function() {
-        timeoutRequest.resolve();
-      }, 5000);
-
-      return resolveStatus.promise;
-    }
-
     function constructURI(url) {
       var uri = document.createElement('a');
 
@@ -382,25 +360,25 @@
 
   <script type="text/template" id="infoTemplate">
     <div>
-      <p><b>{{ relay.uri.hostname }}</b> <span ng-if="relay.status.options['provided-by']">provided by <u>{{ relay.status.options['provided-by'] }}</u></span></p>
-      <div ng-if="relay.status">
-        <span ng-if="relay.status.startTime">Start time: {{ relay.status.startTime | date:"medium" }}</br></span>
-        <span ng-if="relay.status.bytesProxied != undefined">Proxied: {{ relay.status.bytesProxied | bytes }}</br></span>
-        <span ng-if="relay.status.numActiveSessions != undefined">Sessions: {{ relay.status.numActiveSessions }}</br></span>
-        <span ng-if="relay.status.numConnections != undefined">Clients: {{ relay.status.numConnections }}</br></span>
-        <span ng-if="relay.status.options.pools">Pools: {{ relay.status.options.pools.join(', ') }}</br></span>
-        <span ng-if="relay.status.options['global-rate'] != undefined">
-          <span ng-if="relay.status.options['global-rate'] > 0">Global rate limit: {{ relay.status.options['global-rate'] | bytes }}/s</span>
-          <span ng-if="relay.status.options['global-rate'] == 0">Global rate limit: unlimited</span>
-          </br>
+      <p><b>{{ relay.uri.hostname }}</b> <span ng-if="relay.stats.options['provided-by']">provided by <u>{{ relay.stats.options['provided-by'] }}</u></span></p>
+      <div ng-if="relay.stats">
+        <span ng-if="relay.stats.startTime">Start time: {{ relay.stats.startTime | date:"medium" }}</br></span>
+        <span ng-if="relay.stats.bytesProxied != undefined">Proxied: {{ relay.stats.bytesProxied | bytes }}</br></span>
+        <span ng-if="relay.stats.numActiveSessions != undefined">Sessions: {{ relay.stats.numActiveSessions }}</br></span>
+        <span ng-if="relay.stats.numConnections != undefined">Clients: {{ relay.stats.numConnections }}</br></span>
+        <span ng-if="relay.stats.options.pools">Pools: {{ relay.stats.options.pools.join(', ') }}</br></span>
+        <span ng-if="relay.stats.options['global-rate'] != undefined">
+          <span ng-if="relay.stats.options['global-rate'] > 0">Global rate limit: {{ relay.stats.options['global-rate'] | bytes }}/s</span>
+          <span ng-if="relay.stats.options['global-rate'] == 0">Global rate limit: unlimited</span>
+          <br/>
         </span>
-        <span ng-if="relay.status.options['per-session-rate'] != undefined">
-          <span ng-if="relay.status.options['per-session-rate'] > 0">Session rate limit: {{ relay.status.options['per-session-rate'] | bytes }}/s</span>
-          <span ng-if="relay.status.options['per-session-rate'] == 0">Session rate limit: unlimited</span>
-          </br>
+        <span ng-if="relay.stats.options['per-session-rate'] != undefined">
+          <span ng-if="relay.stats.options['per-session-rate'] > 0">Session rate limit: {{ relay.stats.options['per-session-rate'] | bytes }}/s</span>
+          <span ng-if="relay.stats.options['per-session-rate'] == 0">Session rate limit: unlimited</span>
+          <br/>
         </span>
       </div>
-      <div ng-if="!relay.status">
+      <div ng-if="!relay.stats">
         Data unavailable.
       <div>
     </div>

cmd/strelaypoolsrv/main.go

@@ -18,12 +18,16 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/golang/groupcache/lru"
 	"github.com/oschwald/geoip2-golang"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
 	"github.com/syncthing/syncthing/lib/relay/client"
 	"github.com/syncthing/syncthing/lib/sync"
@@ -34,12 +38,42 @@ import (
 type location struct {
 	Latitude  float64 `json:"latitude"`
 	Longitude float64 `json:"longitude"`
+	City      string  `json:"city"`
+	Country   string  `json:"country"`
+	Continent string  `json:"continent"`
 }
 
 type relay struct {
-	URL      string   `json:"url"`
-	Location location `json:"location"`
-	uri      *url.URL
+	URL            string   `json:"url"`
+	Location       location `json:"location"`
+	uri            *url.URL
+	Stats          *stats    `json:"stats"`
+	StatsRetrieved time.Time `json:"statsRetrieved"`
+}
+
+type stats struct {
+	StartTime          time.Time `json:"startTime"`
+	UptimeSeconds      int       `json:"uptimeSeconds"`
+	PendingSessionKeys int       `json:"numPendingSessionKeys"`
+	ActiveSessions     int       `json:"numActiveSessions"`
+	Connections        int       `json:"numConnections"`
+	Proxies            int       `json:"numProxies"`
+	BytesProxied       int       `json:"bytesProxied"`
+	GoVersion          string    `json:"goVersion"`
+	GoOS               string    `json:"goOS"`
+	GoArch             string    `json:"goArch"`
+	GoMaxProcs         int       `json:"goMaxProcs"`
+	GoRoutines         int       `json:"goNumRoutine"`
+	Rates              []int64   `json:"kbps10s1m5m15m30m60m"`
+	Options            struct {
+		NetworkTimeout int      `json:"network-timeout"`
+		PintInterval   int      `json:"ping-interval"`
+		MessageTimeout int      `json:"message-timeout"`
+		SessionRate    int      `json:"per-session-rate"`
+		GlobalRate     int      `json:"global-rate"`
+		Pools          []string `json:"pools"`
+		ProvidedBy     string   `json:"provided-by"`
+	} `json:"options"`
 }
 
 func (r relay) String() string {
@@ -47,9 +81,9 @@ func (r relay) String() string {
 }
 
 type request struct {
-	relay  relay
-	uri    *url.URL
-	result chan result
+	relay      *relay
+	result     chan result
+	queueTimer *prometheus.Timer
 }
 
 type result struct {
@@ -58,23 +92,25 @@ type result struct {
 }
 
 var (
-	testCert       tls.Certificate
-	listen         = ":80"
-	dir            string
-	evictionTime   = time.Hour
-	debug          bool
-	getLRUSize     = 10 << 10
-	getLimitBurst  = 10
-	getLimitAvg    = 1
-	postLRUSize    = 1 << 10
-	postLimitBurst = 2
-	postLimitAvg   = 1
-	getLimit       time.Duration
-	postLimit      time.Duration
-	permRelaysFile string
-	ipHeader       string
-	geoipPath      string
-	proto          string
+	testCert        tls.Certificate
+	knownRelaysFile = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
+	listen          = ":80"
+	dir             string
+	evictionTime    = time.Hour
+	debug           bool
+	getLRUSize      = 10 << 10
+	getLimitBurst   = 10
+	getLimitAvg     = 2
+	postLRUSize     = 1 << 10
+	postLimitBurst  = 2
+	postLimitAvg    = 2
+	getLimit        time.Duration
+	postLimit       time.Duration
+	permRelaysFile  string
+	ipHeader        string
+	geoipPath       string
+	proto           string
+	statsRefresh    = time.Minute / 2
 
 	getMut      = sync.NewRWMutex()
 	getLRUCache *lru.Cache
@@ -85,26 +121,31 @@ var (
 	requests = make(chan request, 10)
 
 	mut             = sync.NewRWMutex()
-	knownRelays     = make([]relay, 0)
-	permanentRelays = make([]relay, 0)
+	knownRelays     = make([]*relay, 0)
+	permanentRelays = make([]*relay, 0)
 	evictionTimers  = make(map[string]*time.Timer)
 )
 
+const (
+	httpStatusEnhanceYourCalm = 429
+)
+
 func main() {
 	flag.StringVar(&listen, "listen", listen, "Listen address")
 	flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
 	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
 	flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
 	flag.IntVar(&getLRUSize, "get-limit-cache", getLRUSize, "Get request limiter cache size")
-	flag.IntVar(&getLimitAvg, "get-limit-avg", 2, "Allowed average get request rate, per 10 s")
+	flag.IntVar(&getLimitAvg, "get-limit-avg", getLimitAvg, "Allowed average get request rate, per 10 s")
 	flag.IntVar(&getLimitBurst, "get-limit-burst", getLimitBurst, "Allowed burst get requests")
 	flag.IntVar(&postLRUSize, "post-limit-cache", postLRUSize, "Post request limiter cache size")
-	flag.IntVar(&postLimitAvg, "post-limit-avg", 2, "Allowed average post request rate, per minute")
+	flag.IntVar(&postLimitAvg, "post-limit-avg", postLimitAvg, "Allowed average post request rate, per minute")
 	flag.IntVar(&postLimitBurst, "post-limit-burst", postLimitBurst, "Allowed burst post requests")
 	flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
 	flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
 	flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
 	flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
+	flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
 
 	flag.Parse()
 
@@ -118,13 +159,31 @@ func main() {
 	var err error
 
 	if permRelaysFile != "" {
-		loadPermanentRelays(permRelaysFile)
+		permanentRelays = loadRelays(permRelaysFile)
 	}
 
 	testCert = createTestCertificate()
 
 	go requestProcessor()
 
+	// Load relays from cache in the background.
+	// Load them in a serial fashion to make sure any genuine requests
+	// are not dropped.
+	go func() {
+		for _, relay := range loadRelays(knownRelaysFile) {
+			resultChan := make(chan result)
+			requests <- request{relay, resultChan, nil}
+			result := <-resultChan
+			if result.err != nil {
+				relayTestsTotal.WithLabelValues("failed").Inc()
+			} else {
+				relayTestsTotal.WithLabelValues("success").Inc()
+			}
+		}
+		// Run the the stats refresher once the relays are loaded.
+		statsRefresher(statsRefresh)
+	}()
+
 	if dir != "" {
 		if debug {
 			log.Println("Starting TLS listener on", listen)
@@ -169,6 +228,7 @@ func main() {
 	handler := http.NewServeMux()
 	handler.HandleFunc("/", handleAssets)
 	handler.HandleFunc("/endpoint", handleRequest)
+	handler.HandleFunc("/metrics", handleMetrics)
 
 	srv := http.Server{
 		Handler:     handler,
@@ -181,6 +241,15 @@ func main() {
 	}
 }
 
+func handleMetrics(w http.ResponseWriter, r *http.Request) {
+	timer := prometheus.NewTimer(metricsRequestsSeconds)
+	// Acquire the mutex just to make sure we're not caught mid-way stats collection
+	mut.RLock()
+	promhttp.Handler().ServeHTTP(w, r)
+	mut.RUnlock()
+	timer.ObserveDuration()
+}
+
 func handleAssets(w http.ResponseWriter, r *http.Request) {
 	assets := auto.Assets()
 	path := r.URL.Path[1:]
@@ -241,6 +310,15 @@ func mimeTypeForFile(file string) string {
 }
 
 func handleRequest(w http.ResponseWriter, r *http.Request) {
+	timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
+
+	lw := NewLoggingResponseWriter(w)
+
+	defer func() {
+		timer.ObserveDuration()
+		apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
+	}()
+
 	if ipHeader != "" {
 		r.RemoteAddr = r.Header.Get(ipHeader)
 	}
@@ -248,13 +326,13 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 	switch r.Method {
 	case "GET":
 		if limit(r.RemoteAddr, getLRUCache, getMut, getLimit, getLimitBurst) {
-			w.WriteHeader(429)
+			w.WriteHeader(httpStatusEnhanceYourCalm)
 			return
 		}
 		handleGetRequest(w, r)
 	case "POST":
 		if limit(r.RemoteAddr, postLRUCache, postMut, postLimit, postLimitBurst) {
-			w.WriteHeader(429)
+			w.WriteHeader(httpStatusEnhanceYourCalm)
 			return
 		}
 		handlePostRequest(w, r)
@@ -278,7 +356,7 @@ func handleGetRequest(w http.ResponseWriter, r *http.Request) {
 		relays[i], relays[j] = relays[j], relays[i]
 	}
 
-	json.NewEncoder(w).Encode(map[string][]relay{
+	json.NewEncoder(w).Encode(map[string][]*relay{
 		"relays": relays,
 	})
 }
@@ -315,13 +393,9 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Get the IP address of the client
-	rhost, _, err := net.SplitHostPort(r.RemoteAddr)
-	if err != nil {
-		if debug {
-			log.Println("Failed to split remote address", r.RemoteAddr)
-		}
-		http.Error(w, err.Error(), 500)
-		return
+	rhost := r.RemoteAddr
+	if host, _, err := net.SplitHostPort(rhost); err == nil {
+		rhost = host
 	}
 
 	ip := net.ParseIP(host)
@@ -333,18 +407,18 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		if debug {
 			log.Println("IP address advertised does not match client IP address", r.RemoteAddr, uri)
 		}
-		http.Error(w, "IP address does not match client IP", http.StatusUnauthorized)
+		http.Error(w, fmt.Sprintf("IP advertised %s does not match client IP %s", host, rhost), http.StatusUnauthorized)
 		return
 	}
+
 	newRelay.uri = uri
-	newRelay.Location = getLocation(uri.Host)
 
 	for _, current := range permanentRelays {
 		if current.uri.Host == newRelay.uri.Host {
 			if debug {
 				log.Println("Asked to add a relay", newRelay, "which exists in permanent list")
 			}
-			http.Error(w, "Invalid request", 500)
+			http.Error(w, "Invalid request", http.StatusBadRequest)
 			return
 		}
 	}
@@ -352,78 +426,105 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 	reschan := make(chan result)
 
 	select {
-	case requests <- request{newRelay, uri, reschan}:
+	case requests <- request{&newRelay, reschan, prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("queue"))}:
 		result := <-reschan
 		if result.err != nil {
-			http.Error(w, result.err.Error(), 500)
+			relayTestsTotal.WithLabelValues("failed").Inc()
+			http.Error(w, result.err.Error(), http.StatusBadRequest)
 			return
 		}
+		relayTestsTotal.WithLabelValues("success").Inc()
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		json.NewEncoder(w).Encode(map[string]time.Duration{
 			"evictionIn": result.eviction,
 		})
 
 	default:
+		relayTestsTotal.WithLabelValues("dropped").Inc()
 		if debug {
 			log.Println("Dropping request")
 		}
-		w.WriteHeader(429)
+		w.WriteHeader(httpStatusEnhanceYourCalm)
 	}
 }
 
 func requestProcessor() {
 	for request := range requests {
-		if debug {
-			log.Println("Request for", request.relay)
-		}
-		if !client.TestRelay(request.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
-			if debug {
-				log.Println("Test for relay", request.relay, "failed")
-			}
-			request.result <- result{fmt.Errorf("test failed"), 0}
-			continue
+		if request.queueTimer != nil {
+			request.queueTimer.ObserveDuration()
 		}
 
-		mut.Lock()
-		timer, ok := evictionTimers[request.relay.uri.Host]
-		if ok {
-			if debug {
-				log.Println("Stopping existing timer for", request.relay)
-			}
-			timer.Stop()
-		}
-
-		for i, current := range knownRelays {
-			if current.uri.Host == request.relay.uri.Host {
-				if debug {
-					log.Println("Relay", request.relay, "already exists")
-				}
-
-				// Evict the old entry anyway, as configuration might have changed.
-				last := len(knownRelays) - 1
-				knownRelays[i] = knownRelays[last]
-				knownRelays = knownRelays[:last]
-
-				goto found
-			}
-		}
-
-		if debug {
-			log.Println("Adding new relay", request.relay)
-		}
-
-	found:
-
-		knownRelays = append(knownRelays, request.relay)
-
-		evictionTimers[request.relay.uri.Host] = time.AfterFunc(evictionTime, evict(request.relay))
-		mut.Unlock()
-		request.result <- result{nil, evictionTime}
+		timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test"))
+		handleRelayTest(request)
+		timer.ObserveDuration()
 	}
-
 }
 
-func evict(relay relay) func() {
+func handleRelayTest(request request) {
+	if debug {
+		log.Println("Request for", request.relay)
+	}
+	if !client.TestRelay(request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
+		if debug {
+			log.Println("Test for relay", request.relay, "failed")
+		}
+		request.result <- result{fmt.Errorf("connection test failed"), 0}
+		return
+	}
+
+	stats := fetchStats(request.relay)
+	location := getLocation(request.relay.uri.Host)
+
+	mut.Lock()
+	if stats != nil {
+		updateMetrics(request.relay.uri.Host, stats, location)
+	}
+	request.relay.Stats = stats
+	request.relay.StatsRetrieved = time.Now()
+	request.relay.Location = location
+
+	timer, ok := evictionTimers[request.relay.uri.Host]
+	if ok {
+		if debug {
+			log.Println("Stopping existing timer for", request.relay)
+		}
+		timer.Stop()
+	}
+
+	for i, current := range knownRelays {
+		if current.uri.Host == request.relay.uri.Host {
+			if debug {
+				log.Println("Relay", request.relay, "already exists")
+			}
+
+			// Evict the old entry anyway, as configuration might have changed.
+			last := len(knownRelays) - 1
+			knownRelays[i] = knownRelays[last]
+			knownRelays = knownRelays[:last]
+
+			goto found
+		}
+	}
+
+	if debug {
+		log.Println("Adding new relay", request.relay)
+	}
+
+found:
+
+	knownRelays = append(knownRelays, request.relay)
+	evictionTimers[request.relay.uri.Host] = time.AfterFunc(evictionTime, evict(request.relay))
+
+	mut.Unlock()
+
+	if err := saveRelays(knownRelaysFile, knownRelays); err != nil {
+		log.Println("Failed to write known relays: " + err.Error())
+	}
+
+	request.result <- result{nil, evictionTime}
+}
+
+func evict(relay *relay) func() {
 	return func() {
 		mut.Lock()
 		defer mut.Unlock()
@@ -438,6 +539,7 @@ func evict(relay relay) func() {
 				last := len(knownRelays) - 1
 				knownRelays[i] = knownRelays[last]
 				knownRelays = knownRelays[:last]
+				deleteMetrics(current.uri.Host)
 			}
 		}
 		delete(evictionTimers, relay.uri.Host)
@@ -445,13 +547,12 @@ func evict(relay relay) func() {
 }
 
 func limit(addr string, cache *lru.Cache, lock sync.RWMutex, intv time.Duration, burst int) bool {
-	host, _, err := net.SplitHostPort(addr)
-	if err != nil {
-		return false
+	if host, _, err := net.SplitHostPort(addr); err == nil {
+		addr = host
 	}
 
 	lock.RLock()
-	bkt, ok := cache.Get(host)
+	bkt, ok := cache.Get(addr)
 	lock.RUnlock()
 	if ok {
 		bkt := bkt.(*rate.Limiter)
@@ -461,18 +562,20 @@ func limit(addr string, cache *lru.Cache, lock sync.RWMutex, intv time.Duration,
 		}
 	} else {
 		lock.Lock()
-		cache.Add(host, rate.NewLimiter(rate.Every(intv), burst))
+		cache.Add(addr, rate.NewLimiter(rate.Every(intv), burst))
 		lock.Unlock()
 	}
 	return false
 }
 
-func loadPermanentRelays(file string) {
+func loadRelays(file string) []*relay {
 	content, err := ioutil.ReadFile(file)
 	if err != nil {
-		log.Fatal(err)
+		log.Println("Failed to load relays: " + err.Error())
+		return nil
 	}
 
+	var relays []*relay
 	for _, line := range strings.Split(string(content), "\n") {
 		if len(line) == 0 {
 			continue
@@ -481,21 +584,30 @@ func loadPermanentRelays(file string) {
 		uri, err := url.Parse(line)
 		if err != nil {
 			if debug {
-				log.Println("Skipping permanent relay", line, "due to parse error", err)
+				log.Println("Skipping relay", line, "due to parse error", err)
 			}
 			continue
 
 		}
 
-		permanentRelays = append(permanentRelays, relay{
+		relays = append(relays, &relay{
 			URL:      line,
 			Location: getLocation(uri.Host),
 			uri:      uri,
 		})
 		if debug {
-			log.Println("Adding permanent relay", line)
+			log.Println("Adding relay", line)
 		}
 	}
+	return relays
+}
+
+func saveRelays(file string, relays []*relay) error {
+	var content string
+	for _, relay := range relays {
+		content += relay.uri.String() + "\n"
+	}
+	return ioutil.WriteFile(file, []byte(content), 0777)
 }
 
 func createTestCertificate() tls.Certificate {
@@ -514,6 +626,8 @@ func createTestCertificate() tls.Certificate {
 }
 
 func getLocation(host string) location {
+	timer := prometheus.NewTimer(locationLookupSeconds)
+	defer timer.ObserveDuration()
 	db, err := geoip2.Open(geoipPath)
 	if err != nil {
 		return location{}
@@ -531,7 +645,24 @@ func getLocation(host string) location {
 	}
 
 	return location{
-		Latitude:  city.Location.Latitude,
 		Longitude: city.Location.Longitude,
+		Latitude:  city.Location.Latitude,
+		City:      city.City.Names["en"],
+		Country:   city.Country.IsoCode,
+		Continent: city.Continent.Code,
 	}
 }
+
+type loggingResponseWriter struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
+	return &loggingResponseWriter{w, http.StatusOK}
+}
+
+func (lrw *loggingResponseWriter) WriteHeader(code int) {
+	lrw.statusCode = code
+	lrw.ResponseWriter.WriteHeader(code)
+}

cmd/strelaypoolsrv/stats.go

@@ -0,0 +1,213 @@
+// Copyright (C) 2018 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"encoding/json"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+func init() {
+	prometheus.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "syncthing_relaypoolsrv"))
+}
+
+var (
+	statusClient = http.Client{
+		Timeout: 5 * time.Second,
+	}
+
+	apiRequestsTotal   = makeCounter("api_requests_total", "Number of API requests.", "type", "result")
+	apiRequestsSeconds = makeSummary("api_requests_seconds", "Latency of API requests.", "type")
+
+	relayTestsTotal         = makeCounter("tests_total", "Number of relay tests.", "result")
+	relayTestActionsSeconds = makeSummary("test_actions_seconds", "Latency of relay test actions.", "type")
+
+	locationLookupSeconds = makeSummary("location_lookup_seconds", "Latency of location lookups.").WithLabelValues()
+
+	metricsRequestsSeconds = makeSummary("metrics_requests_seconds", "Latency of metric requests.").WithLabelValues()
+	scrapeSeconds          = makeSummary("relay_scrape_seconds", "Latency of metric scrapes from remote relays.", "result")
+
+	relayUptime             = makeGauge("relay_uptime", "Uptime of relay", "relay")
+	relayPendingSessionKeys = makeGauge("relay_pending_session_keys", "Number of pending session keys (two keys per session, one per each side of the connection)", "relay")
+	relayActiveSessions     = makeGauge("relay_active_sessions", "Number of sessions that are happening, a session contains two parties", "relay")
+	relayConnections        = makeGauge("relay_connections", "Number of devices connected to the relay", "relay")
+	relayProxies            = makeGauge("relay_proxies", "Number of active proxy routines sending data between peers (two proxies per session, one for each way)", "relay")
+	relayBytesProxied       = makeGauge("relay_bytes_proxied", "Number of bytes proxied by the relay", "relay")
+	relayGoRoutines         = makeGauge("relay_go_routines", "Number of Go routines in the process", "relay")
+	relaySessionRate        = makeGauge("relay_session_rate", "Rate applied per session", "relay")
+	relayGlobalRate         = makeGauge("relay_global_rate", "Global rate applied on the whole relay", "relay")
+	relayBuildInfo          = makeGauge("relay_build_info", "Build information about a relay", "relay", "go_version", "go_os", "go_arch")
+	relayLocationInfo       = makeGauge("relay_location_info", "Location information about a relay", "relay", "city", "country", "continent")
+)
+
+func makeGauge(name string, help string, labels ...string) *prometheus.GaugeVec {
+	gauge := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "syncthing",
+			Subsystem: "relaypoolsrv",
+			Name:      name,
+			Help:      help,
+		},
+		labels,
+	)
+	prometheus.MustRegister(gauge)
+	return gauge
+}
+
+func makeSummary(name string, help string, labels ...string) *prometheus.SummaryVec {
+	summary := prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Namespace:  "syncthing",
+			Subsystem:  "relaypoolsrv",
+			Name:       name,
+			Help:       help,
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		labels,
+	)
+	prometheus.MustRegister(summary)
+	return summary
+}
+
+func makeCounter(name string, help string, labels ...string) *prometheus.CounterVec {
+	counter := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "relaypoolsrv",
+			Name:      name,
+			Help:      help,
+		},
+		labels,
+	)
+	prometheus.MustRegister(counter)
+	return counter
+}
+
+func statsRefresher(interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	for range ticker.C {
+		refreshStats()
+	}
+}
+
+type statsFetchResult struct {
+	relay *relay
+	stats *stats
+}
+
+func refreshStats() {
+	mut.RLock()
+	relays := append(permanentRelays, knownRelays...)
+	mut.RUnlock()
+
+	now := time.Now()
+	wg := sync.NewWaitGroup()
+
+	results := make(chan statsFetchResult, len(relays))
+	for _, rel := range relays {
+		wg.Add(1)
+		go func(rel *relay) {
+			t0 := time.Now()
+			stats := fetchStats(rel)
+			duration := time.Now().Sub(t0).Seconds()
+			result := "success"
+			if stats == nil {
+				result = "failed"
+			}
+			scrapeSeconds.WithLabelValues(result).Observe(duration)
+
+			results <- statsFetchResult{
+				relay: rel,
+				stats: fetchStats(rel),
+			}
+			wg.Done()
+		}(rel)
+	}
+
+	wg.Wait()
+	close(results)
+
+	mut.Lock()
+	relayBuildInfo.Reset()
+	relayLocationInfo.Reset()
+	for result := range results {
+		result.relay.StatsRetrieved = now
+		result.relay.Stats = result.stats
+		if result.stats == nil {
+			deleteMetrics(result.relay.uri.Host)
+		} else {
+			updateMetrics(result.relay.uri.Host, result.stats, result.relay.Location)
+		}
+	}
+	mut.Unlock()
+}
+
+func fetchStats(relay *relay) *stats {
+	statusAddr := relay.uri.Query().Get("statusAddr")
+	if statusAddr == "" {
+		statusAddr = ":22070"
+	}
+
+	statusHost, statusPort, err := net.SplitHostPort(statusAddr)
+	if err != nil {
+		return nil
+	}
+
+	if statusHost == "" {
+		if host, _, err := net.SplitHostPort(relay.uri.Host); err != nil {
+			return nil
+		} else {
+			statusHost = host
+		}
+	}
+
+	url := "http://" + net.JoinHostPort(statusHost, statusPort) + "/status"
+
+	response, err := statusClient.Get(url)
+	if err != nil {
+		return nil
+	}
+
+	var stats stats
+
+	if json.NewDecoder(response.Body).Decode(&stats); err != nil {
+		return nil
+	}
+	return &stats
+}
+
+func updateMetrics(host string, stats *stats, location location) {
+	if stats.GoVersion != "" || stats.GoOS != "" || stats.GoArch != "" {
+		relayBuildInfo.WithLabelValues(host, stats.GoVersion, stats.GoOS, stats.GoArch).Add(1)
+	}
+	if location.City != "" || location.Country != "" || location.Continent != "" {
+		relayLocationInfo.WithLabelValues(host, location.City, location.Country, location.Continent).Add(1)
+	}
+	relayUptime.WithLabelValues(host).Set(float64(stats.UptimeSeconds))
+	relayPendingSessionKeys.WithLabelValues(host).Set(float64(stats.PendingSessionKeys))
+	relayActiveSessions.WithLabelValues(host).Set(float64(stats.ActiveSessions))
+	relayConnections.WithLabelValues(host).Set(float64(stats.Connections))
+	relayProxies.WithLabelValues(host).Set(float64(stats.Proxies))
+	relayBytesProxied.WithLabelValues(host).Set(float64(stats.BytesProxied))
+	relayGoRoutines.WithLabelValues(host).Set(float64(stats.GoRoutines))
+	relaySessionRate.WithLabelValues(host).Set(float64(stats.Options.SessionRate))
+	relayGlobalRate.WithLabelValues(host).Set(float64(stats.Options.GlobalRate))
+}
+
+func deleteMetrics(host string) {
+	relayUptime.DeleteLabelValues(host)
+	relayPendingSessionKeys.DeleteLabelValues(host)
+	relayActiveSessions.DeleteLabelValues(host)
+	relayConnections.DeleteLabelValues(host)
+	relayProxies.DeleteLabelValues(host)
+	relayBytesProxied.DeleteLabelValues(host)
+	relayGoRoutines.DeleteLabelValues(host)
+	relaySessionRate.DeleteLabelValues(host)
+	relayGlobalRate.DeleteLabelValues(host)
+}

cmd/strelaysrv/README.md

@@ -1,18 +1,12 @@
 strelaysrv
 ==========
 
-[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/strelaysrv.svg?style=flat-square)](http://build.syncthing.net/job/strelaysrv/lastBuild/)
-
 This is the relay server for the `syncthing` project.
 
-To get it, run `go get github.com/syncthing/syncthing/cmd/strelaysrv` or download the
-[latest build](http://build.syncthing.net/job/strelaysrv/lastSuccessfulBuild/artifact/)
-from the build server.
-
 :exclamation:Warnings:exclamation: - Read or regret
 -----
 
-By default, all relay servers will join to the default public relay pool, which means that the relay server will be availble for public use, and **will consume your bandwidth** helping others to connect.
+By default, all relay servers will join to the default public relay pool, which means that the relay server will be available for public use, and **will consume your bandwidth** helping others to connect.
 
 If you wish to disable this behaviour, please specify the `-pools=""` argument.
 

cmd/strelaysrv/listener.go

@@ -59,6 +59,13 @@ func listener(proto, addr string, config *tls.Config) {
 
 func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
 	conn := tls.Server(tcpConn, config)
+	if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil {
+		if debug {
+			log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr())
+		}
+		conn.Close()
+		return
+	}
 	err := conn.Handshake()
 	if err != nil {
 		if debug {
@@ -81,6 +88,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
 		conn.Close()
 		return
 	}
+	conn.SetDeadline(time.Time{})
 
 	id := syncthingprotocol.NewDeviceID(certs[0].Raw)
 
@@ -96,7 +104,9 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
 	go messageReader(conn, messages, errors)
 
 	pingTicker := time.NewTicker(pingInterval)
+	defer pingTicker.Stop()
 	timeoutTicker := time.NewTimer(networkTimeout)
+	defer timeoutTicker.Stop()
 	joined := false
 
 	for {
@@ -277,6 +287,7 @@ func sessionConnectionHandler(conn net.Conn) {
 		if debug {
 			log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr())
 		}
+		conn.Close()
 		return
 	}
 

cmd/strelaysrv/main.go

@@ -64,12 +64,13 @@ var (
 
 	limitCheckTimer *time.Timer
 
-	sessionLimitBps int
-	globalLimitBps  int
-	overLimit       int32
-	descriptorLimit int64
-	sessionLimiter  *rate.Limiter
-	globalLimiter   *rate.Limiter
+	sessionLimitBps   int
+	globalLimitBps    int
+	overLimit         int32
+	descriptorLimit   int64
+	sessionLimiter    *rate.Limiter
+	globalLimiter     *rate.Limiter
+	networkBufferSize int
 
 	statusAddr       string
 	poolAddrs        string
@@ -81,8 +82,16 @@ var (
 	natLease   int
 	natRenewal int
 	natTimeout int
+
+	pprofEnabled bool
 )
 
+// httpClient is the HTTP client we use for outbound requests. It has a
+// timeout and may get further options set during initialization.
+var httpClient = &http.Client{
+	Timeout: 30 * time.Second,
+}
+
 func main() {
 	log.SetFlags(log.Lshortfile | log.LstdFlags)
 
@@ -105,6 +114,8 @@ func main() {
 	flag.IntVar(&natLease, "nat-lease", 60, "NAT lease length in minutes")
 	flag.IntVar(&natRenewal, "nat-renewal", 30, "NAT renewal frequency in minutes")
 	flag.IntVar(&natTimeout, "nat-timeout", 10, "NAT discovery timeout in seconds")
+	flag.BoolVar(&pprofEnabled, "pprof", false, "Enable the built in profiling on the status server")
+	flag.IntVar(&networkBufferSize, "network-buffer", 2048, "Network buffer size (two of these per proxied connection)")
 	flag.Parse()
 
 	if extAddress == "" {
@@ -124,14 +135,14 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+
 	if laddr.IP != nil && !laddr.IP.IsUnspecified() {
+		// We bind to a specific address. Our outgoing HTTP requests should
+		// also come from that address.
 		laddr.Port = 0
-		transport, ok := http.DefaultTransport.(*http.Transport)
-		if ok {
-			transport.Dial = (&net.Dialer{
-				Timeout:   30 * time.Second,
-				LocalAddr: laddr,
-			}).Dial
+		boundDialer := &net.Dialer{LocalAddr: laddr}
+		httpClient.Transport = &http.Transport{
+			DialContext: boundDialer.DialContext,
 		}
 	}
 

cmd/strelaysrv/pool.go

@@ -7,7 +7,6 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"log"
-	"net/http"
 	"net/url"
 	"time"
 )
@@ -27,7 +26,7 @@ func poolHandler(pool string, uri *url.URL, mapping mapping) {
 			uriCopy.String(),
 		})
 
-		resp, err := http.Post(pool, "application/json", &b)
+		resp, err := httpClient.Post(pool, "application/json", &b)
 		if err != nil {
 			log.Println("Error joining pool", pool, err)
 		} else if resp.StatusCode == 500 {

cmd/strelaysrv/session.go

@@ -189,7 +189,7 @@ done:
 	// We can end up here in 3 cases:
 	// 1. Timeout joining, in which case there are potentially entries in pendingSessions
 	// 2. General session end/timeout, in which case there are entries in activeSessions
-	// 3. Protocol handler calls dropSession as one of it's clients disconnects.
+	// 3. Protocol handler calls dropSession as one of its clients disconnects.
 
 	sessionMut.Lock()
 	delete(pendingSessions, string(s.serverkey))
@@ -254,7 +254,7 @@ func (s *session) proxy(c1, c2 net.Conn) error {
 	atomic.AddInt64(&numProxies, 1)
 	defer atomic.AddInt64(&numProxies, -1)
 
-	buf := make([]byte, 65536)
+	buf := make([]byte, networkBufferSize)
 	for {
 		c1.SetReadDeadline(time.Now().Add(networkTimeout))
 		n, err := c1.Read(buf)

cmd/strelaysrv/status.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"log"
 	"net/http"
+	"net/http/pprof"
 	"runtime"
 	"sync/atomic"
 	"time"
@@ -16,8 +17,19 @@ var rc *rateCalculator
 func statusService(addr string) {
 	rc = newRateCalculator(360, 10*time.Second, &bytesProxied)
 
-	http.HandleFunc("/status", getStatus)
-	if err := http.ListenAndServe(addr, nil); err != nil {
+	handler := http.NewServeMux()
+	handler.HandleFunc("/status", getStatus)
+	if pprofEnabled {
+		handler.HandleFunc("/debug/pprof/", pprof.Index)
+	}
+
+	srv := http.Server{
+		Addr:        addr,
+		Handler:     handler,
+		ReadTimeout: 15 * time.Second,
+	}
+	srv.SetKeepAlivesEnabled(false)
+	if err := srv.ListenAndServe(); err != nil {
 		log.Fatal(err)
 	}
 }
@@ -28,6 +40,10 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 
 	sessionMut.Lock()
 	// This can potentially be double the number of pending sessions, as each session has two keys, one for each side.
+	status["version"] = Version
+	status["buildHost"] = BuildHost
+	status["buildUser"] = BuildUser
+	status["buildDate"] = BuildDate
 	status["startTime"] = rc.startTime
 	status["uptimeSeconds"] = time.Since(rc.startTime) / time.Second
 	status["numPendingSessionKeys"] = len(pendingSessions)

cmd/stsigtool/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -24,7 +24,7 @@ func main() {
 	flag.Parse()
 
 	if flag.NArg() < 1 {
-		log.Println(`Usage:
+		log.Print(`Usage:
 	stsigtool <command>
 
 Where command is one of:
@@ -40,6 +40,7 @@ Where command is one of:
 
 	verify <signaturefile> <datafile> <pubkeyfile>
 		- verify a signature, using the specified public key file
+
 `)
 	}
 

cmd/stvanity/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/stwatchfile/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/auditservice.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/auditservice_test.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -20,13 +20,13 @@ func TestAuditService(t *testing.T) {
 	service := newAuditService(buf)
 
 	// Event sent before start, will not be logged
-	events.Default.Log(events.Ping, "the first event")
+	events.Default.Log(events.ConfigSaved, "the first event")
 
 	go service.Serve()
 	service.WaitForStart()
 
 	// Event that should end up in the audit log
-	events.Default.Log(events.Ping, "the second event")
+	events.Default.Log(events.ConfigSaved, "the second event")
 
 	// We need to give the events time to arrive, since the channels are buffered etc.
 	time.Sleep(10 * time.Millisecond)
@@ -35,7 +35,7 @@ func TestAuditService(t *testing.T) {
 	service.WaitForStop()
 
 	// This event should not be logged, since we have stopped.
-	events.Default.Log(events.Ping, "the third event")
+	events.Default.Log(events.ConfigSaved, "the third event")
 
 	result := string(buf.Bytes())
 	t.Log(result)

cmd/syncthing/blockprof.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/buildtag_noupgrade.go

@@ -0,0 +1,13 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//+build noupgrade
+
+package main
+
+func init() {
+	BuildTags = append(BuildTags, "noupgrade")
+}

cmd/syncthing/buildtag_race.go

@@ -0,0 +1,13 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//+build race
+
+package main
+
+func init() {
+	BuildTags = append(BuildTags, "race")
+}

cmd/syncthing/cpuusage.go

@@ -0,0 +1,59 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"math"
+	"time"
+
+	metrics "github.com/rcrowley/go-metrics"
+)
+
+const cpuTickRate = 5 * time.Second
+
+type cpuService struct {
+	avg  metrics.EWMA
+	stop chan struct{}
+}
+
+func newCPUService() *cpuService {
+	return &cpuService{
+		// 10 second average. Magic alpha value comes from looking at EWMA package
+		// definitions of EWMA1, EWMA5. The tick rate *must* be five seconds (hard
+		// coded in the EWMA package).
+		avg:  metrics.NewEWMA(1 - math.Exp(-float64(cpuTickRate)/float64(time.Second)/10.0)),
+		stop: make(chan struct{}),
+	}
+}
+
+func (s *cpuService) Serve() {
+	// Initialize prevUsage to an actual value returned by cpuUsage
+	// instead of zero, because at least Windows returns a huge negative
+	// number here that then slowly increments...
+	prevUsage := cpuUsage()
+	ticker := time.NewTicker(cpuTickRate)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			curUsage := cpuUsage()
+			s.avg.Update(int64((curUsage - prevUsage) / time.Millisecond))
+			prevUsage = curUsage
+			s.avg.Tick()
+		case <-s.stop:
+			return
+		}
+	}
+}
+
+func (s *cpuService) Stop() {
+	close(s.stop)
+}
+
+func (s *cpuService) Rate() float64 {
+	return s.avg.Rate()
+}

cmd/syncthing/cpuusage_solaris.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 //+build solaris
 
@@ -59,40 +59,20 @@ type prusage_t struct {
 
 }
 
-func solarisPrusage(pid int, rusage *prusage_t) error {
-	fd, err := os.Open(fmt.Sprintf("/proc/%d/usage", pid))
+var procFile = fmt.Sprintf("/proc/%d/usage", os.Getpid())
+
+func cpuUsage() time.Duration {
+	fd, err := os.Open(procFile)
 	if err != nil {
-		return err
+		return 0
 	}
+
+	var rusage prusage_t
 	err = binary.Read(fd, binary.LittleEndian, rusage)
 	fd.Close()
-	return err
-}
-
-func init() {
-	go trackCPUUsage()
-}
-
-func trackCPUUsage() {
-	var prevUsage int64
-	var prevTime = time.Now().UnixNano()
-	var rusage prusage_t
-	var pid = os.Getpid()
-	for range time.NewTicker(time.Second).C {
-		err := solarisPrusage(pid, &rusage)
-		if err != nil {
-			l.Warnln("getting prusage:", err)
-			continue
-		}
-		curTime := time.Now().UnixNano()
-		timeDiff := curTime - prevTime
-		curUsage := rusage.Pr_utime.Nano() + rusage.Pr_stime.Nano()
-		usageDiff := curUsage - prevUsage
-		cpuUsageLock.Lock()
-		copy(cpuUsagePercent[1:], cpuUsagePercent[0:])
-		cpuUsagePercent[0] = 100 * float64(usageDiff) / float64(timeDiff)
-		cpuUsageLock.Unlock()
-		prevTime = curTime
-		prevUsage = curUsage
+	if err != nil {
+		return 0
 	}
+
+	return time.Duration(rusage.Pr_utime.Nano() + rusage.Pr_stime.Nano())
 }

cmd/syncthing/cpuusage_unix.go

@@ -0,0 +1,18 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//+build !windows,!solaris
+
+package main
+
+import "syscall"
+import "time"
+
+func cpuUsage() time.Duration {
+	var rusage syscall.Rusage
+	syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
+	return time.Duration(rusage.Utime.Nano() + rusage.Stime.Nano())
+}

cmd/syncthing/cpuusage_windows.go

@@ -0,0 +1,27 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//+build windows
+
+package main
+
+import "syscall"
+import "time"
+
+func cpuUsage() time.Duration {
+	handle, err := syscall.GetCurrentProcess()
+	if err != nil {
+		return 0
+	}
+	defer syscall.CloseHandle(handle)
+
+	var ctime, etime, ktime, utime syscall.Filetime
+	if err := syscall.GetProcessTimes(handle, &ctime, &etime, &ktime, &utime); err != nil {
+		return 0
+	}
+
+	return time.Duration(ktime.Nanoseconds() + utime.Nanoseconds())
+}

cmd/syncthing/debug.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/gui.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -13,9 +13,11 @@ import (
 	"io/ioutil"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"reflect"
+	"regexp"
 	"runtime"
 	"runtime/pprof"
 	"sort"
@@ -25,24 +27,35 @@ import (
 
 	"github.com/rcrowley/go-metrics"
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/connections"
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/discover"
 	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/logger"
 	"github.com/syncthing/syncthing/lib/model"
-	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/stats"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/upgrade"
+	"github.com/syncthing/syncthing/lib/versioner"
 	"github.com/vitrun/qart/qr"
 	"golang.org/x/crypto/bcrypt"
 )
 
 var (
 	startTime = time.Now()
+
+	// matches a bcrypt hash and not too much else
+	bcryptExpr = regexp.MustCompile(`^\$2[aby]\$\d+\$.{50,}`)
+)
+
+const (
+	defaultEventMask   = events.AllEvents &^ events.LocalChangeDetected &^ events.RemoteChangeDetected
+	diskEventMask      = events.LocalChangeDetected | events.RemoteChangeDetected
+	eventSubBufferSize = 1000
 )
 
 type apiService struct {
@@ -52,8 +65,8 @@ type apiService struct {
 	httpsKeyFile       string
 	statics            *staticsServer
 	model              modelIntf
-	eventSub           events.BufferedSubscription
-	diskEventSub       events.BufferedSubscription
+	eventSubs          map[events.EventType]events.BufferedSubscription
+	eventSubsMut       sync.Mutex
 	discoverer         discover.CachingMux
 	connectionsService connectionsIntf
 	fss                *folderSummaryService
@@ -62,6 +75,7 @@ type apiService struct {
 	configChanged      chan struct{} // signals intentional listener close due to config change
 	started            chan string   // signals startup complete by sending the listener address, for testing only
 	startedOnce        chan struct{} // the service has started successfully at least once
+	cpu                rater
 
 	guiErrors logger.Recorder
 	systemLog logger.Recorder
@@ -71,7 +85,8 @@ type modelIntf interface {
 	GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{}
 	Completion(device protocol.DeviceID, folder string) model.FolderCompletion
 	Override(folder string)
-	NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated, int)
+	NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated)
+	RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error)
 	NeedSize(folder string) db.Counts
 	ConnectionStats() map[string]interface{}
 	DeviceStatistics() map[string]stats.DeviceStatistics
@@ -81,29 +96,35 @@ type modelIntf interface {
 	ResetFolder(folder string)
 	Availability(folder, file string, version protocol.Vector, block protocol.BlockInfo) []model.Availability
 	GetIgnores(folder string) ([]string, []string, error)
+	GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error)
+	RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error)
 	SetIgnores(folder string, content []string) error
 	DelayScan(folder string, next time.Duration)
 	ScanFolder(folder string) error
 	ScanFolders() map[string]error
 	ScanFolderSubdirs(folder string, subs []string) error
 	BringToFront(folder, file string)
-	ConnectedTo(deviceID protocol.DeviceID) bool
+	Connection(deviceID protocol.DeviceID) (connections.Connection, bool)
 	GlobalSize(folder string) db.Counts
 	LocalSize(folder string) db.Counts
 	CurrentSequence(folder string) (int64, bool)
 	RemoteSequence(folder string) (int64, bool)
 	State(folder string) (string, time.Time, error)
+	UsageReportingStats(version int, preview bool) map[string]interface{}
+	PullErrors(folder string) ([]model.FileError, error)
+	WatchError(folder string) error
 }
 
 type configIntf interface {
 	GUI() config.GUIConfiguration
 	RawCopy() config.Configuration
 	Options() config.OptionsConfiguration
-	Replace(cfg config.Configuration) error
+	Replace(cfg config.Configuration) (config.Waiter, error)
 	Subscribe(c config.Committer)
 	Folders() map[string]config.FolderConfiguration
 	Devices() map[protocol.DeviceID]config.DeviceConfiguration
-	SetDevice(config.DeviceConfiguration) error
+	SetDevice(config.DeviceConfiguration) (config.Waiter, error)
+	SetDevices([]config.DeviceConfiguration) (config.Waiter, error)
 	Save() error
 	ListenAddresses() []string
 	RequiresRestart() bool
@@ -111,18 +132,26 @@ type configIntf interface {
 
 type connectionsIntf interface {
 	Status() map[string]interface{}
+	NATType() string
 }
 
-func newAPIService(id protocol.DeviceID, cfg configIntf, httpsCertFile, httpsKeyFile, assetDir string, m modelIntf, eventSub events.BufferedSubscription, diskEventSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService connectionsIntf, errors, systemLog logger.Recorder) *apiService {
+type rater interface {
+	Rate() float64
+}
+
+func newAPIService(id protocol.DeviceID, cfg configIntf, httpsCertFile, httpsKeyFile, assetDir string, m modelIntf, defaultSub, diskSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService connectionsIntf, errors, systemLog logger.Recorder, cpu rater) *apiService {
 	service := &apiService{
-		id:                 id,
-		cfg:                cfg,
-		httpsCertFile:      httpsCertFile,
-		httpsKeyFile:       httpsKeyFile,
-		statics:            newStaticsServer(cfg.GUI().Theme, assetDir),
-		model:              m,
-		eventSub:           eventSub,
-		diskEventSub:       diskEventSub,
+		id:            id,
+		cfg:           cfg,
+		httpsCertFile: httpsCertFile,
+		httpsKeyFile:  httpsKeyFile,
+		statics:       newStaticsServer(cfg.GUI().Theme, assetDir),
+		model:         m,
+		eventSubs: map[events.EventType]events.BufferedSubscription{
+			defaultEventMask: defaultSub,
+			diskEventMask:    diskSub,
+		},
+		eventSubsMut:       sync.NewMutex(),
 		discoverer:         discoverer,
 		connectionsService: connectionsService,
 		systemConfigMut:    sync.NewMutex(),
@@ -131,6 +160,7 @@ func newAPIService(id protocol.DeviceID, cfg configIntf, httpsCertFile, httpsKey
 		startedOnce:        make(chan struct{}),
 		guiErrors:          errors,
 		systemLog:          systemLog,
+		cpu:                cpu,
 	}
 
 	return service
@@ -231,10 +261,13 @@ func (s *apiService) Serve() {
 	getRestMux.HandleFunc("/rest/db/file", s.getDBFile)                          // folder file
 	getRestMux.HandleFunc("/rest/db/ignores", s.getDBIgnores)                    // folder
 	getRestMux.HandleFunc("/rest/db/need", s.getDBNeed)                          // folder [perpage] [page]
+	getRestMux.HandleFunc("/rest/db/remoteneed", s.getDBRemoteNeed)              // device folder [perpage] [page]
 	getRestMux.HandleFunc("/rest/db/status", s.getDBStatus)                      // folder
 	getRestMux.HandleFunc("/rest/db/browse", s.getDBBrowse)                      // folder [prefix] [dirsonly] [levels]
-	getRestMux.HandleFunc("/rest/events", s.getIndexEvents)                      // since [limit]
-	getRestMux.HandleFunc("/rest/events/disk", s.getDiskEvents)                  // since [limit]
+	getRestMux.HandleFunc("/rest/folder/versions", s.getFolderVersions)          // folder
+	getRestMux.HandleFunc("/rest/folder/pullerrors", s.getPullErrors)            // folder
+	getRestMux.HandleFunc("/rest/events", s.getIndexEvents)                      // [since] [limit] [timeout] [events]
+	getRestMux.HandleFunc("/rest/events/disk", s.getDiskEvents)                  // [since] [limit] [timeout]
 	getRestMux.HandleFunc("/rest/stats/device", s.getDeviceStats)                // -
 	getRestMux.HandleFunc("/rest/stats/folder", s.getFolderStats)                // -
 	getRestMux.HandleFunc("/rest/svc/deviceid", s.getDeviceID)                   // id
@@ -261,6 +294,7 @@ func (s *apiService) Serve() {
 	postRestMux.HandleFunc("/rest/db/ignores", s.postDBIgnores)                    // folder
 	postRestMux.HandleFunc("/rest/db/override", s.postDBOverride)                  // folder
 	postRestMux.HandleFunc("/rest/db/scan", s.postDBScan)                          // folder [sub...] [delay]
+	postRestMux.HandleFunc("/rest/folder/versions", s.postFolderVersionsRestore)   // folder <body>
 	postRestMux.HandleFunc("/rest/system/config", s.postSystemConfig)              // <body>
 	postRestMux.HandleFunc("/rest/system/error", s.postSystemError)                // <body>
 	postRestMux.HandleFunc("/rest/system/error/clear", s.postSystemErrorClear)     // -
@@ -269,8 +303,8 @@ func (s *apiService) Serve() {
 	postRestMux.HandleFunc("/rest/system/restart", s.postSystemRestart)            // -
 	postRestMux.HandleFunc("/rest/system/shutdown", s.postSystemShutdown)          // -
 	postRestMux.HandleFunc("/rest/system/upgrade", s.postSystemUpgrade)            // -
-	postRestMux.HandleFunc("/rest/system/pause", s.makeDevicePauseHandler(true))   // device
-	postRestMux.HandleFunc("/rest/system/resume", s.makeDevicePauseHandler(false)) // device
+	postRestMux.HandleFunc("/rest/system/pause", s.makeDevicePauseHandler(true))   // [device]
+	postRestMux.HandleFunc("/rest/system/resume", s.makeDevicePauseHandler(false)) // [device]
 	postRestMux.HandleFunc("/rest/system/debug", s.postSystemDebug)                // [enable] [disable]
 
 	// Debug endpoints, not for general use
@@ -316,7 +350,7 @@ func (s *apiService) Serve() {
 	}
 
 	// Add the CORS handling
-	handler = corsMiddleware(handler)
+	handler = corsMiddleware(handler, guiCfg.InsecureAllowFrameLoading)
 
 	if addressIsLocalhost(guiCfg.Address()) && !guiCfg.InsecureSkipHostCheck {
 		// Verify source host
@@ -343,7 +377,7 @@ func (s *apiService) Serve() {
 		s.started <- listener.Addr().String()
 	}
 
-	// Indicate successfull initial startup, to ourselves and to interested
+	// Indicate successful initial startup, to ourselves and to interested
 	// listeners (i.e. the thing that starts the browser).
 	select {
 	case <-s.startedOnce:
@@ -443,7 +477,7 @@ func debugMiddleware(h http.Handler) http.Handler {
 	})
 }
 
-func corsMiddleware(next http.Handler) http.Handler {
+func corsMiddleware(next http.Handler, allowFrameLoading bool) http.Handler {
 	// Handle CORS headers and CORS OPTIONS request.
 	// CORS OPTIONS request are typically sent by browser during AJAX preflight
 	// when the browser initiate a POST request.
@@ -470,6 +504,27 @@ func corsMiddleware(next http.Handler) http.Handler {
 			return
 		}
 
+		// Other security related headers that should be present.
+		// https://www.owasp.org/index.php/Security_Headers
+
+		if !allowFrameLoading {
+			// We don't want to be rendered in an <iframe>,
+			// <frame> or <object>. (Unless we do it ourselves.
+			// This is also an escape hatch for people who serve
+			// Syncthing GUI as part of their own website
+			// through a proxy, so they don't need to set the
+			// allowFrameLoading bool.)
+			w.Header().Set("X-Frame-Options", "SAMEORIGIN")
+		}
+
+		// If the browser senses an XSS attack it's allowed to take
+		// action. (How this would not always be the default I
+		// don't fully understand.)
+		w.Header().Set("X-XSS-Protection", "1; mode=block")
+
+		// Our content type headers are correct. Don't guess.
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+
 		// For everything else, pass to the next handler
 		next.ServeHTTP(w, r)
 		return
@@ -613,24 +668,39 @@ func (s *apiService) getDBCompletion(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	comp := s.model.Completion(device, folder)
-	sendJSON(w, map[string]interface{}{
+	sendJSON(w, jsonCompletion(s.model.Completion(device, folder)))
+}
+
+func jsonCompletion(comp model.FolderCompletion) map[string]interface{} {
+	return map[string]interface{}{
 		"completion":  comp.CompletionPct,
 		"needBytes":   comp.NeedBytes,
+		"needItems":   comp.NeedItems,
 		"globalBytes": comp.GlobalBytes,
 		"needDeletes": comp.NeedDeletes,
-	})
+	}
 }
 
 func (s *apiService) getDBStatus(w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
 	folder := qs.Get("folder")
-	sendJSON(w, folderSummary(s.cfg, s.model, folder))
+	if sum, err := folderSummary(s.cfg, s.model, folder); err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+	} else {
+		sendJSON(w, sum)
+	}
 }
 
-func folderSummary(cfg configIntf, m modelIntf, folder string) map[string]interface{} {
+func folderSummary(cfg configIntf, m modelIntf, folder string) (map[string]interface{}, error) {
 	var res = make(map[string]interface{})
 
+	pullErrors, err := m.PullErrors(folder)
+	if err != nil && err != model.ErrFolderPaused {
+		// Stats from the db can still be obtained if the folder is just paused
+		return nil, err
+	}
+	res["pullErrors"] = len(pullErrors)
+
 	res["invalid"] = "" // Deprecated, retains external API for now
 
 	global := m.GlobalSize(folder)
@@ -644,7 +714,6 @@ func folderSummary(cfg configIntf, m modelIntf, folder string) map[string]interf
 
 	res["inSyncFiles"], res["inSyncBytes"] = global.Files-need.Files, global.Bytes-need.Bytes
 
-	var err error
 	res["state"], res["stateChanged"], err = m.State(folder)
 	if err != nil {
 		res["error"] = err.Error()
@@ -665,7 +734,12 @@ func folderSummary(cfg configIntf, m modelIntf, folder string) map[string]interf
 		}
 	}
 
-	return res
+	err = m.WatchError(folder)
+	if err != nil {
+		res["watchError"] = err.Error()
+	}
+
+	return res, nil
 }
 
 func (s *apiService) postDBOverride(w http.ResponseWriter, r *http.Request) {
@@ -674,11 +748,7 @@ func (s *apiService) postDBOverride(w http.ResponseWriter, r *http.Request) {
 	go s.model.Override(folder)
 }
 
-func (s *apiService) getDBNeed(w http.ResponseWriter, r *http.Request) {
-	qs := r.URL.Query()
-
-	folder := qs.Get("folder")
-
+func getPagingParams(qs url.Values) (int, int) {
 	page, err := strconv.Atoi(qs.Get("page"))
 	if err != nil || page < 1 {
 		page = 1
@@ -687,20 +757,52 @@ func (s *apiService) getDBNeed(w http.ResponseWriter, r *http.Request) {
 	if err != nil || perpage < 1 {
 		perpage = 1 << 16
 	}
+	return page, perpage
+}
 
-	progress, queued, rest, total := s.model.NeedFolderFiles(folder, page, perpage)
+func (s *apiService) getDBNeed(w http.ResponseWriter, r *http.Request) {
+	qs := r.URL.Query()
+
+	folder := qs.Get("folder")
+
+	page, perpage := getPagingParams(qs)
+
+	progress, queued, rest := s.model.NeedFolderFiles(folder, page, perpage)
 
 	// Convert the struct to a more loose structure, and inject the size.
 	sendJSON(w, map[string]interface{}{
-		"progress": s.toNeedSlice(progress),
-		"queued":   s.toNeedSlice(queued),
-		"rest":     s.toNeedSlice(rest),
-		"total":    total,
+		"progress": toNeedSlice(progress),
+		"queued":   toNeedSlice(queued),
+		"rest":     toNeedSlice(rest),
 		"page":     page,
 		"perpage":  perpage,
 	})
 }
 
+func (s *apiService) getDBRemoteNeed(w http.ResponseWriter, r *http.Request) {
+	qs := r.URL.Query()
+
+	folder := qs.Get("folder")
+	device := qs.Get("device")
+	deviceID, err := protocol.DeviceIDFromString(device)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	page, perpage := getPagingParams(qs)
+
+	if files, err := s.model.RemoteNeedFolderFiles(deviceID, folder, page, perpage); err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+	} else {
+		sendJSON(w, map[string]interface{}{
+			"files":   toNeedSlice(files),
+			"page":    page,
+			"perpage": perpage,
+		})
+	}
+}
+
 func (s *apiService) getSystemConnections(w http.ResponseWriter, r *http.Request) {
 	sendJSON(w, s.model.ConnectionStats())
 }
@@ -751,7 +853,7 @@ func (s *apiService) postSystemConfig(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if to.GUI.Password != s.cfg.GUI().Password {
-		if to.GUI.Password != "" {
+		if to.GUI.Password != "" && !bcryptExpr.MatchString(to.GUI.Password) {
 			hash, err := bcrypt.GenerateFromPassword([]byte(to.GUI.Password), 0)
 			if err != nil {
 				l.Warnln("bcrypting password:", err)
@@ -763,21 +865,9 @@ func (s *apiService) postSystemConfig(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	// Fixup usage reporting settings
-
-	if curAcc := s.cfg.Options().URAccepted; to.Options.URAccepted > curAcc {
-		// UR was enabled
-		to.Options.URAccepted = usageReportVersion
-		to.Options.URUniqueID = rand.String(8)
-	} else if to.Options.URAccepted < curAcc {
-		// UR was disabled
-		to.Options.URAccepted = -1
-		to.Options.URUniqueID = ""
-	}
-
 	// Activate and save
 
-	if err := s.cfg.Replace(to); err != nil {
+	if _, err := s.cfg.Replace(to); err != nil {
 		l.Warnln("Replacing config:", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
@@ -836,14 +926,11 @@ func (s *apiService) flushResponse(resp string, w http.ResponseWriter) {
 	f.Flush()
 }
 
-var cpuUsagePercent [10]float64 // The last ten seconds
-var cpuUsageLock = sync.NewRWMutex()
-
 func (s *apiService) getSystemStatus(w http.ResponseWriter, r *http.Request) {
 	var m runtime.MemStats
 	runtime.ReadMemStats(&m)
 
-	tilde, _ := osutil.ExpandTilde("~")
+	tilde, _ := fs.ExpandTilde("~")
 	res := make(map[string]interface{})
 	res["myID"] = myID.String()
 	res["goroutines"] = runtime.NumGoroutine()
@@ -865,15 +952,11 @@ func (s *apiService) getSystemStatus(w http.ResponseWriter, r *http.Request) {
 	}
 
 	res["connectionServiceStatus"] = s.connectionsService.Status()
-
-	cpuUsageLock.RLock()
-	var cpusum float64
-	for _, p := range cpuUsagePercent {
-		cpusum += p
-	}
-	cpuUsageLock.RUnlock()
-	res["cpuPercent"] = cpusum / float64(len(cpuUsagePercent)) / float64(runtime.NumCPU())
+	// cpuUsage.Rate() is in milliseconds per second, so dividing by ten
+	// gives us percent
+	res["cpuPercent"] = s.cpu.Rate() / 10 / float64(runtime.NumCPU())
 	res["pathSeparator"] = string(filepath.Separator)
+	res["urVersionMax"] = usageReportVersion
 	res["uptime"] = int(time.Since(startTime).Seconds())
 	res["startTime"] = startTime
 
@@ -952,7 +1035,11 @@ func (s *apiService) getSystemDiscovery(w http.ResponseWriter, r *http.Request)
 }
 
 func (s *apiService) getReport(w http.ResponseWriter, r *http.Request) {
-	sendJSON(w, reportData(s.cfg, s.model))
+	version := usageReportVersion
+	if val, _ := strconv.Atoi(r.URL.Query().Get("version")); val > 0 {
+		version = val
+	}
+	sendJSON(w, reportData(s.cfg, s.model, s.connectionsService, version, true))
 }
 
 func (s *apiService) getRandomString(w http.ResponseWriter, r *http.Request) {
@@ -968,7 +1055,9 @@ func (s *apiService) getRandomString(w http.ResponseWriter, r *http.Request) {
 func (s *apiService) getDBIgnores(w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
 
-	ignores, patterns, err := s.model.GetIgnores(qs.Get("folder"))
+	folder := qs.Get("folder")
+
+	ignores, patterns, err := s.model.GetIgnores(folder)
 	if err != nil {
 		http.Error(w, err.Error(), 500)
 		return
@@ -1008,20 +1097,29 @@ func (s *apiService) postDBIgnores(w http.ResponseWriter, r *http.Request) {
 
 func (s *apiService) getIndexEvents(w http.ResponseWriter, r *http.Request) {
 	s.fss.gotEventRequest()
-	s.getEvents(w, r, s.eventSub)
+	mask := s.getEventMask(r.URL.Query().Get("events"))
+	sub := s.getEventSub(mask)
+	s.getEvents(w, r, sub)
 }
 
 func (s *apiService) getDiskEvents(w http.ResponseWriter, r *http.Request) {
-	s.getEvents(w, r, s.diskEventSub)
+	sub := s.getEventSub(diskEventMask)
+	s.getEvents(w, r, sub)
 }
 
 func (s *apiService) getEvents(w http.ResponseWriter, r *http.Request, eventSub events.BufferedSubscription) {
 	qs := r.URL.Query()
 	sinceStr := qs.Get("since")
 	limitStr := qs.Get("limit")
+	timeoutStr := qs.Get("timeout")
 	since, _ := strconv.Atoi(sinceStr)
 	limit, _ := strconv.Atoi(limitStr)
 
+	timeout := defaultEventTimeout
+	if timeoutSec, timeoutErr := strconv.Atoi(timeoutStr); timeoutErr == nil && timeoutSec >= 0 { // 0 is a valid timeout
+		timeout = time.Duration(timeoutSec) * time.Second
+	}
+
 	// Flush before blocking, to indicate that we've received the request and
 	// that it should not be retried. Must set Content-Type header before
 	// flushing.
@@ -1029,7 +1127,8 @@ func (s *apiService) getEvents(w http.ResponseWriter, r *http.Request, eventSub
 	f := w.(http.Flusher)
 	f.Flush()
 
-	evs := eventSub.Since(since, nil)
+	// If there are no events available return an empty slice, as this gets serialized as `[]`
+	evs := eventSub.Since(since, []events.Event{}, timeout)
 	if 0 < limit && limit < len(evs) {
 		evs = evs[len(evs)-limit:]
 	}
@@ -1037,12 +1136,38 @@ func (s *apiService) getEvents(w http.ResponseWriter, r *http.Request, eventSub
 	sendJSON(w, evs)
 }
 
+func (s *apiService) getEventMask(evs string) events.EventType {
+	eventMask := defaultEventMask
+	if evs != "" {
+		eventList := strings.Split(evs, ",")
+		eventMask = 0
+		for _, ev := range eventList {
+			eventMask |= events.UnmarshalEventType(strings.TrimSpace(ev))
+		}
+	}
+	return eventMask
+}
+
+func (s *apiService) getEventSub(mask events.EventType) events.BufferedSubscription {
+	s.eventSubsMut.Lock()
+	bufsub, ok := s.eventSubs[mask]
+	if !ok {
+		evsub := events.Default.Subscribe(mask)
+		bufsub = events.NewBufferedSubscription(evsub, eventSubBufferSize)
+		s.eventSubs[mask] = bufsub
+	}
+	s.eventSubsMut.Unlock()
+
+	return bufsub
+}
+
 func (s *apiService) getSystemUpgrade(w http.ResponseWriter, r *http.Request) {
-	if noUpgrade {
+	if noUpgradeFromEnv {
 		http.Error(w, upgrade.ErrUpgradeUnsupported.Error(), 500)
 		return
 	}
-	rel, err := upgrade.LatestRelease(s.cfg.Options().ReleasesURL, Version)
+	opts := s.cfg.Options()
+	rel, err := upgrade.LatestRelease(opts.ReleasesURL, Version, opts.UpgradeToPreReleases)
 	if err != nil {
 		http.Error(w, err.Error(), 500)
 		return
@@ -1083,7 +1208,8 @@ func (s *apiService) getLang(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *apiService) postSystemUpgrade(w http.ResponseWriter, r *http.Request) {
-	rel, err := upgrade.LatestRelease(s.cfg.Options().ReleasesURL, Version)
+	opts := s.cfg.Options()
+	rel, err := upgrade.LatestRelease(opts.ReleasesURL, Version, opts.UpgradeToPreReleases)
 	if err != nil {
 		l.Warnln("getting latest release:", err)
 		http.Error(w, err.Error(), 500)
@@ -1109,19 +1235,31 @@ func (s *apiService) makeDevicePauseHandler(paused bool) http.HandlerFunc {
 		var qs = r.URL.Query()
 		var deviceStr = qs.Get("device")
 
-		device, err := protocol.DeviceIDFromString(deviceStr)
-		if err != nil {
-			http.Error(w, err.Error(), 500)
-			return
+		var cfgs []config.DeviceConfiguration
+
+		if deviceStr == "" {
+			for _, cfg := range s.cfg.Devices() {
+				cfg.Paused = paused
+				cfgs = append(cfgs, cfg)
+			}
+		} else {
+			device, err := protocol.DeviceIDFromString(deviceStr)
+			if err != nil {
+				http.Error(w, err.Error(), 500)
+				return
+			}
+
+			cfg, ok := s.cfg.Devices()[device]
+			if !ok {
+				http.Error(w, "not found", http.StatusNotFound)
+				return
+			}
+
+			cfg.Paused = paused
+			cfgs = append(cfgs, cfg)
 		}
 
-		cfg, ok := s.cfg.Devices()[device]
-		if !ok {
-			http.Error(w, "not found", http.StatusNotFound)
-		}
-
-		cfg.Paused = paused
-		if err := s.cfg.SetDevice(cfg); err != nil {
+		if _, err := s.cfg.SetDevices(cfgs); err != nil {
 			http.Error(w, err.Error(), 500)
 		}
 	}
@@ -1131,18 +1269,17 @@ func (s *apiService) postDBScan(w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
 	folder := qs.Get("folder")
 	if folder != "" {
+		subs := qs["sub"]
+		err := s.model.ScanFolderSubdirs(folder, subs)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
 		nextStr := qs.Get("next")
 		next, err := strconv.Atoi(nextStr)
 		if err == nil {
 			s.model.DelayScan(folder, time.Duration(next)*time.Second)
 		}
-
-		subs := qs["sub"]
-		err = s.model.ScanFolderSubdirs(folder, subs)
-		if err != nil {
-			http.Error(w, err.Error(), 500)
-			return
-		}
 	} else {
 		errors := s.model.ScanFolders()
 		if len(errors) > 0 {
@@ -1181,7 +1318,7 @@ func (s *apiService) getPeerCompletion(w http.ResponseWriter, r *http.Request) {
 	for _, folder := range s.cfg.Folders() {
 		for _, device := range folder.DeviceIDs() {
 			deviceStr := device.String()
-			if s.model.ConnectedTo(device) {
+			if _, ok := s.model.Connection(device); ok {
 				tot[deviceStr] += s.model.Completion(device, folder.ID).CompletionPct
 			} else {
 				tot[deviceStr] = 0
@@ -1198,32 +1335,117 @@ func (s *apiService) getPeerCompletion(w http.ResponseWriter, r *http.Request) {
 	sendJSON(w, comp)
 }
 
-func (s *apiService) getSystemBrowse(w http.ResponseWriter, r *http.Request) {
+func (s *apiService) getFolderVersions(w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
-	current := qs.Get("current")
-	if current == "" {
-		if roots, err := osutil.GetFilesystemRoots(); err == nil {
-			sendJSON(w, roots)
-		} else {
-			http.Error(w, err.Error(), 500)
-		}
+	versions, err := s.model.GetFolderVersions(qs.Get("folder"))
+	if err != nil {
+		http.Error(w, err.Error(), 500)
 		return
 	}
-	search, _ := osutil.ExpandTilde(current)
-	pathSeparator := string(os.PathSeparator)
-	if strings.HasSuffix(current, pathSeparator) && !strings.HasSuffix(search, pathSeparator) {
-		search = search + pathSeparator
+	sendJSON(w, versions)
+}
+
+func (s *apiService) postFolderVersionsRestore(w http.ResponseWriter, r *http.Request) {
+	qs := r.URL.Query()
+
+	bs, err := ioutil.ReadAll(r.Body)
+	r.Body.Close()
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
 	}
-	subdirectories, _ := osutil.Glob(search + "*")
-	ret := make([]string, 0, len(subdirectories))
-	for _, subdirectory := range subdirectories {
-		info, err := os.Stat(subdirectory)
-		if err == nil && info.IsDir() {
-			ret = append(ret, subdirectory+pathSeparator)
+
+	var versions map[string]time.Time
+	err = json.Unmarshal(bs, &versions)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	ferr, err := s.model.RestoreFolderVersions(qs.Get("folder"), versions)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+	sendJSON(w, ferr)
+}
+
+func (s *apiService) getPullErrors(w http.ResponseWriter, r *http.Request) {
+	qs := r.URL.Query()
+	folder := qs.Get("folder")
+	page, perpage := getPagingParams(qs)
+
+	errors, err := s.model.PullErrors(folder)
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+		return
+	}
+
+	start := (page - 1) * perpage
+	if start >= len(errors) {
+		errors = nil
+	} else {
+		errors = errors[start:]
+		if perpage < len(errors) {
+			errors = errors[:perpage]
 		}
 	}
 
-	sendJSON(w, ret)
+	sendJSON(w, map[string]interface{}{
+		"folder":  folder,
+		"errors":  errors,
+		"page":    page,
+		"perpage": perpage,
+	})
+}
+
+func (s *apiService) getSystemBrowse(w http.ResponseWriter, r *http.Request) {
+	qs := r.URL.Query()
+	current := qs.Get("current")
+
+	// Default value or in case of error unmarshalling ends up being basic fs.
+	var fsType fs.FilesystemType
+	fsType.UnmarshalText([]byte(qs.Get("filesystem")))
+
+	sendJSON(w, browseFiles(current, fsType))
+}
+
+func browseFiles(current string, fsType fs.FilesystemType) []string {
+	if current == "" {
+		filesystem := fs.NewFilesystem(fsType, "")
+		if roots, err := filesystem.Roots(); err == nil {
+			return roots
+		}
+		return nil
+	}
+	search, _ := fs.ExpandTilde(current)
+	pathSeparator := string(fs.PathSeparator)
+
+	if strings.HasSuffix(current, pathSeparator) && !strings.HasSuffix(search, pathSeparator) {
+		search = search + pathSeparator
+	}
+	searchDir := filepath.Dir(search)
+
+	// The searchFile should be the last component of search, or empty if it
+	// ends with a path separator
+	var searchFile string
+	if !strings.HasSuffix(search, pathSeparator) {
+		searchFile = filepath.Base(search)
+	}
+
+	fs := fs.NewFilesystem(fsType, searchDir)
+
+	subdirectories, _ := fs.Glob(searchFile + "*")
+
+	ret := make([]string, 0, len(subdirectories))
+	for _, subdirectory := range subdirectories {
+		info, err := fs.Stat(subdirectory)
+		if err == nil && info.IsDir() {
+			ret = append(ret, filepath.Join(searchDir, subdirectory)+pathSeparator)
+		}
+	}
+	return ret
 }
 
 func (s *apiService) getCPUProf(w http.ResponseWriter, r *http.Request) {
@@ -1252,7 +1474,7 @@ func (s *apiService) getHeapProf(w http.ResponseWriter, r *http.Request) {
 	pprof.WriteHeapProfile(w)
 }
 
-func (s *apiService) toNeedSlice(fs []db.FileInfoTruncated) []jsonDBFileInfo {
+func toNeedSlice(fs []db.FileInfoTruncated) []jsonDBFileInfo {
 	res := make([]jsonDBFileInfo, len(fs))
 	for i, f := range fs {
 		res[i] = jsonDBFileInfo(f)
@@ -1274,6 +1496,7 @@ func (f jsonFileInfo) MarshalJSON() ([]byte, error) {
 		"invalid":       f.Invalid,
 		"noPermissions": f.NoPermissions,
 		"modified":      protocol.FileInfo(f).ModTime(),
+		"modifiedBy":    f.ModifiedBy.String(),
 		"sequence":      f.Sequence,
 		"numBlocks":     len(f.Blocks),
 		"version":       jsonVersionVector(f.Version),
@@ -1285,13 +1508,14 @@ type jsonDBFileInfo db.FileInfoTruncated
 func (f jsonDBFileInfo) MarshalJSON() ([]byte, error) {
 	return json.Marshal(map[string]interface{}{
 		"name":          f.Name,
-		"type":          f.Type,
+		"type":          f.Type.String(),
 		"size":          f.Size,
 		"permissions":   fmt.Sprintf("%#o", f.Permissions),
 		"deleted":       f.Deleted,
 		"invalid":       f.Invalid,
 		"noPermissions": f.NoPermissions,
 		"modified":      db.FileInfoTruncated(f).ModTime(),
+		"modifiedBy":    f.ModifiedBy.String(),
 		"sequence":      f.Sequence,
 	})
 }
@@ -1336,9 +1560,14 @@ func addressIsLocalhost(addr string) bool {
 		host = addr
 	}
 	switch strings.ToLower(host) {
-	case "127.0.0.1", "::1", "localhost":
+	case "localhost", "localhost.":
 		return true
 	default:
-		return false
+		ip := net.ParseIP(host)
+		if ip == nil {
+			// not an IP address
+			return false
+		}
+		return ip.IsLoopback()
 	}
 }

cmd/syncthing/gui_auth.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/gui_csrf.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/gui_statics.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -89,6 +89,10 @@ func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) {
 	if s.assetDir != "" {
 		p := filepath.Join(s.assetDir, theme, filepath.FromSlash(file))
 		if _, err := os.Stat(p); err == nil {
+			mtype := s.mimeTypeForFile(file)
+			if len(mtype) != 0 {
+				w.Header().Set("Content-Type", mtype)
+			}
 			http.ServeFile(w, r, p)
 			return
 		}
@@ -101,6 +105,10 @@ func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) {
 		if s.assetDir != "" {
 			p := filepath.Join(s.assetDir, config.DefaultTheme, filepath.FromSlash(file))
 			if _, err := os.Stat(p); err == nil {
+				mtype := s.mimeTypeForFile(file)
+				if len(mtype) != 0 {
+					w.Header().Set("Content-Type", mtype)
+				}
 				http.ServeFile(w, r, p)
 				return
 			}
@@ -141,17 +149,17 @@ func (s *staticsServer) serveThemes(w http.ResponseWriter, r *http.Request) {
 func (s *staticsServer) mimeTypeForFile(file string) string {
 	// We use a built in table of the common types since the system
 	// TypeByExtension might be unreliable. But if we don't know, we delegate
-	// to the system.
+	// to the system. All our files are UTF-8.
 	ext := filepath.Ext(file)
 	switch ext {
 	case ".htm", ".html":
-		return "text/html"
+		return "text/html; charset=utf-8"
 	case ".css":
-		return "text/css"
+		return "text/css; charset=utf-8"
 	case ".js":
-		return "application/javascript"
+		return "application/javascript; charset=utf-8"
 	case ".json":
-		return "application/json"
+		return "application/json; charset=utf-8"
 	case ".png":
 		return "image/png"
 	case ".ttf":
@@ -159,7 +167,7 @@ func (s *staticsServer) mimeTypeForFile(file string) string {
 	case ".woff":
 		return "application/x-font-woff"
 	case ".svg":
-		return "image/svg+xml"
+		return "image/svg+xml; charset=utf-8"
 	default:
 		return mime.TypeByExtension(ext)
 	}

cmd/syncthing/gui_test.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -16,6 +16,8 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"testing"
@@ -23,6 +25,8 @@ import (
 
 	"github.com/d4l3k/messagediff"
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/thejerf/suture"
@@ -70,7 +74,7 @@ func TestStopAfterBrokenConfig(t *testing.T) {
 	}
 	w := config.Wrap("/dev/null", cfg)
 
-	srv := newAPIService(protocol.LocalDeviceID, w, "../../test/h1/https-cert.pem", "../../test/h1/https-key.pem", "", nil, nil, nil, nil, nil, nil, nil)
+	srv := newAPIService(protocol.LocalDeviceID, w, "../../test/h1/https-cert.pem", "../../test/h1/https-key.pem", "", nil, nil, nil, nil, nil, nil, nil, nil)
 	srv.started = make(chan string)
 
 	sup := suture.NewSimple("test")
@@ -474,11 +478,12 @@ func startHTTP(cfg *mockedConfig) (string, error) {
 	connections := new(mockedConnections)
 	errorLog := new(mockedLoggerRecorder)
 	systemLog := new(mockedLoggerRecorder)
+	cpu := new(mockedCPUService)
 	addrChan := make(chan string)
 
 	// Instantiate the API service
 	svc := newAPIService(protocol.LocalDeviceID, cfg, httpsCertFile, httpsKeyFile, assetDir, model,
-		eventSub, diskEventSub, discoverer, connections, errorLog, systemLog)
+		eventSub, diskEventSub, discoverer, connections, errorLog, systemLog, cpu)
 	svc.started = addrChan
 
 	// Actually start the API service
@@ -837,16 +842,24 @@ func TestAddressIsLocalhost(t *testing.T) {
 		// These are all valid localhost addresses
 		{"localhost", true},
 		{"LOCALHOST", true},
+		{"localhost.", true},
 		{"::1", true},
 		{"127.0.0.1", true},
+		{"127.23.45.56", true},
 		{"localhost:8080", true},
 		{"LOCALHOST:8000", true},
+		{"localhost.:8080", true},
 		{"[::1]:8080", true},
 		{"127.0.0.1:8080", true},
+		{"127.23.45.56:8080", true},
 
 		// These are all non-localhost addresses
 		{"example.com", false},
 		{"example.com:8080", false},
+		{"localhost.com", false},
+		{"localhost.com:8080", false},
+		{"www.localhost", false},
+		{"www.localhost:8080", false},
 		{"192.0.2.10", false},
 		{"192.0.2.10:8080", false},
 		{"0.0.0.0", false},
@@ -924,3 +937,92 @@ func TestOptionsRequest(t *testing.T) {
 		t.Fatal("OPTIONS on /rest/system/status should return a 'Access-Control-Allow-Headers: Content-Type, X-API-KEY' header")
 	}
 }
+
+func TestEventMasks(t *testing.T) {
+	cfg := new(mockedConfig)
+	defSub := new(mockedEventSub)
+	diskSub := new(mockedEventSub)
+	svc := newAPIService(protocol.LocalDeviceID, cfg, "", "", "", nil, defSub, diskSub, nil, nil, nil, nil, nil)
+
+	if mask := svc.getEventMask(""); mask != defaultEventMask {
+		t.Errorf("incorrect default mask %x != %x", int64(mask), int64(defaultEventMask))
+	}
+
+	expected := events.FolderSummary | events.LocalChangeDetected
+	if mask := svc.getEventMask("FolderSummary,LocalChangeDetected"); mask != expected {
+		t.Errorf("incorrect parsed mask %x != %x", int64(mask), int64(expected))
+	}
+
+	expected = 0
+	if mask := svc.getEventMask("WeirdEvent,something else that doesn't exist"); mask != expected {
+		t.Errorf("incorrect parsed mask %x != %x", int64(mask), int64(expected))
+	}
+
+	if res := svc.getEventSub(defaultEventMask); res != defSub {
+		t.Errorf("should have returned the given default event sub")
+	}
+	if res := svc.getEventSub(diskEventMask); res != diskSub {
+		t.Errorf("should have returned the given disk event sub")
+	}
+	if res := svc.getEventSub(events.LocalIndexUpdated); res == nil || res == defSub || res == diskSub {
+		t.Errorf("should have returned a valid, non-default event sub")
+	}
+}
+
+func TestBrowse(t *testing.T) {
+	pathSep := string(os.PathSeparator)
+
+	tmpDir, err := ioutil.TempDir("", "syncthing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	if err := os.Mkdir(filepath.Join(tmpDir, "dir"), 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(filepath.Join(tmpDir, "file"), []byte("hello"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	// We expect completion to return the full path to the completed
+	// directory, with an ending slash.
+	dirPath := filepath.Join(tmpDir, "dir") + pathSep
+
+	cases := []struct {
+		current string
+		returns []string
+	}{
+		// The direcotory without slash is completed to one with slash.
+		{tmpDir, []string{tmpDir + pathSep}},
+		// With slash it's completed to its contents.
+		// Dirs are given pathSeps.
+		// Files are not returned.
+		{tmpDir + pathSep, []string{dirPath}},
+		// Globbing is automatic based on prefix.
+		{tmpDir + pathSep + "d", []string{dirPath}},
+		{tmpDir + pathSep + "di", []string{dirPath}},
+		{tmpDir + pathSep + "dir", []string{dirPath}},
+		{tmpDir + pathSep + "f", nil},
+		{tmpDir + pathSep + "q", nil},
+	}
+
+	for _, tc := range cases {
+		ret := browseFiles(tc.current, fs.FilesystemTypeBasic)
+		if !equalStrings(ret, tc.returns) {
+			t.Errorf("browseFiles(%q) => %q, expected %q", tc.current, ret, tc.returns)
+		}
+	}
+}
+
+func equalStrings(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}

cmd/syncthing/gui_unix.go

@@ -1,37 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-//+build !windows,!solaris
-
-package main
-
-import (
-	"syscall"
-	"time"
-)
-
-func init() {
-	go trackCPUUsage()
-}
-
-func trackCPUUsage() {
-	var prevUsage int64
-	var prevTime = time.Now().UnixNano()
-	var rusage syscall.Rusage
-	for range time.NewTicker(time.Second).C {
-		syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
-		curTime := time.Now().UnixNano()
-		timeDiff := curTime - prevTime
-		curUsage := rusage.Utime.Nano() + rusage.Stime.Nano()
-		usageDiff := curUsage - prevUsage
-		cpuUsageLock.Lock()
-		copy(cpuUsagePercent[1:], cpuUsagePercent[0:])
-		cpuUsagePercent[0] = 100 * float64(usageDiff) / float64(timeDiff)
-		cpuUsageLock.Unlock()
-		prevTime = curTime
-		prevUsage = curUsage
-	}
-}

cmd/syncthing/gui_windows.go

@@ -1,58 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-//+build windows
-
-package main
-
-import (
-	"syscall"
-	"time"
-)
-
-func init() {
-	go trackCPUUsage()
-}
-
-func trackCPUUsage() {
-	handle, err := syscall.GetCurrentProcess()
-	if err != nil {
-		l.Warnln("Cannot track CPU usage:", err)
-		return
-	}
-
-	var ctime, etime, ktime, utime syscall.Filetime
-	err = syscall.GetProcessTimes(handle, &ctime, &etime, &ktime, &utime)
-	if err != nil {
-		l.Warnln("Cannot track CPU usage:", err)
-		return
-	}
-
-	prevTime := ctime.Nanoseconds()
-	prevUsage := ktime.Nanoseconds() + utime.Nanoseconds() // Always overflows
-
-	for range time.NewTicker(time.Second).C {
-		err := syscall.GetProcessTimes(handle, &ctime, &etime, &ktime, &utime)
-		if err != nil {
-			continue
-		}
-
-		curTime := time.Now().UnixNano()
-		timeDiff := curTime - prevTime
-		// This is sometimes 0, no clue why.
-		if timeDiff == 0 {
-			continue
-		}
-		curUsage := ktime.Nanoseconds() + utime.Nanoseconds()
-		usageDiff := curUsage - prevUsage
-		cpuUsageLock.Lock()
-		copy(cpuUsagePercent[1:], cpuUsagePercent[0:])
-		cpuUsagePercent[0] = 100 * float64(usageDiff) / float64(timeDiff)
-		cpuUsageLock.Unlock()
-		prevTime = curTime
-		prevUsage = curUsage
-	}
-}

cmd/syncthing/heapprof.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/locations.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -13,7 +13,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/fs"
 )
 
 type locationEnum string
@@ -65,7 +65,7 @@ func expandLocations() error {
 			dir = strings.Replace(dir, "${"+varName+"}", value, -1)
 		}
 		var err error
-		dir, err = osutil.ExpandTilde(dir)
+		dir, err = fs.ExpandTilde(dir)
 		if err != nil {
 			return err
 		}
@@ -86,7 +86,7 @@ func defaultConfigDir() string {
 		return filepath.Join(os.Getenv("AppData"), "Syncthing")
 
 	case "darwin":
-		dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
+		dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
 		if err != nil {
 			l.Fatalln(err)
 		}
@@ -96,7 +96,7 @@ func defaultConfigDir() string {
 		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
 			return filepath.Join(xdgCfg, "syncthing")
 		}
-		dir, err := osutil.ExpandTilde("~/.config/syncthing")
+		dir, err := fs.ExpandTilde("~/.config/syncthing")
 		if err != nil {
 			l.Fatalln(err)
 		}
@@ -106,7 +106,7 @@ func defaultConfigDir() string {
 
 // homeDir returns the user's home directory, or dies trying.
 func homeDir() string {
-	home, err := osutil.ExpandTilde("~")
+	home, err := fs.ExpandTilde("~")
 	if err != nil {
 		l.Fatalln(err)
 	}

cmd/syncthing/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -37,15 +37,16 @@ import (
 	"github.com/syncthing/syncthing/lib/dialer"
 	"github.com/syncthing/syncthing/lib/discover"
 	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/logger"
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/sha256"
-	"github.com/syncthing/syncthing/lib/symlinks"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/upgrade"
+	"github.com/syncthing/syncthing/lib/weakhash"
 
 	"github.com/thejerf/suture"
 
@@ -60,8 +61,10 @@ var (
 	BuildHost         = "unknown"
 	BuildUser         = "unknown"
 	IsRelease         bool
+	IsCandidate       bool
 	IsBeta            bool
 	LongVersion       string
+	BuildTags         []string
 	allowedVersionExp = regexp.MustCompile(`^v\d+\.\d+\.\d+(-[a-z0-9]+)*(\.\d+)*(\+\d+-g[0-9a-f]+)?(-[^\s]+)?$`)
 )
 
@@ -78,19 +81,12 @@ const (
 	tlsDefaultCommonName = "syncthing"
 	httpsRSABits         = 2048
 	bepRSABits           = 0 // 384 bit ECDSA used instead
-	pingEventInterval    = time.Minute
+	defaultEventTimeout  = time.Minute
 	maxSystemErrors      = 5
 	initialSystemLog     = 10
 	maxSystemLog         = 250
 )
 
-// The discovery results are sorted by their source priority.
-const (
-	ipv6LocalDiscoveryPriority = iota
-	ipv4LocalDiscoveryPriority
-	globalDiscoveryPriority
-)
-
 func init() {
 	if Version != "unknown-dev" {
 		// If not a generic dev build, version string should come from git describe
@@ -98,15 +94,26 @@ func init() {
 			l.Fatalf("Invalid version string %q;\n\tdoes not match regexp %v", Version, allowedVersionExp)
 		}
 	}
+}
 
-	// Check for a clean release build. A release is something like "v0.1.2",
-	// with an optional suffix of letters and dot separated numbers like
-	// "-beta3.47". If there's more stuff, like a plus sign and a commit hash
-	// and so on, then it's not a release. If there's a dash anywhere in
-	// there, it's some kind of beta or prerelease version.
+func setBuildMetadata() {
+	// Check for a clean release build. A release is something like
+	// "v0.1.2", with an optional suffix of letters and dot separated
+	// numbers like "-beta3.47". If there's more stuff, like a plus sign and
+	// a commit hash and so on, then it's not a release. If it has a dash in
+	// it, it's some sort of beta, release candidate or special build. If it
+	// has "-rc." in it, like "v0.14.35-rc.42", then it's a candidate build.
+	//
+	// So, every build that is not a stable release build has IsBeta = true.
+	// This is used to enable some extra debugging (the deadlock detector).
+	//
+	// Release candidate builds are also "betas" from this point of view and
+	// will have that debugging enabled. In addition, some features are
+	// forced for release candidates - auto upgrade, and usage reporting.
 
 	exp := regexp.MustCompile(`^v\d+\.\d+\.\d+(-[a-z]+[\d\.]+)?$`)
 	IsRelease = exp.MatchString(Version)
+	IsCandidate = strings.Contains(Version, "-rc.")
 	IsBeta = strings.Contains(Version, "-")
 
 	stamp, _ := strconv.Atoi(BuildStamp)
@@ -114,6 +121,10 @@ func init() {
 
 	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
 	LongVersion = fmt.Sprintf(`syncthing %s "%s" (%s %s-%s) %s@%s %s`, Version, Codename, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
+
+	if len(BuildTags) > 0 {
+		LongVersion = fmt.Sprintf("%s [%s]", LongVersion, strings.Join(BuildTags, ", "))
+	}
 }
 
 var (
@@ -168,14 +179,11 @@ are mostly useful for developers. Use with care.
  STPERFSTATS       Write running performance statistics to perf-$pid.csv. Not
                    supported on Windows.
 
- STDEADLOCK        Used for debugging internal deadlocks. Use only under
-                   direction of a developer.
-
  STDEADLOCKTIMEOUT Used for debugging internal deadlocks; sets debug
                    sensitivity. Use only under direction of a developer.
 
- STDEADLOCKTHRESHOLD Used for debugging internal deadlocks; sets debug
-                     sensitivity.  Use only under direction of a developer.
+ STLOCKTHRESHOLD   Used for debugging internal deadlocks; sets debug
+                   sensitivity.  Use only under direction of a developer.
 
  STNORESTART       Equivalent to the -no-restart argument. Disable the
                    Syncthing monitor process which handles restarts for some
@@ -189,6 +197,11 @@ are mostly useful for developers. Use with care.
                    "minio" for the github.com/minio/sha256-simd implementation,
                    and blank (the default) for auto detection.
 
+ STRECHECKDBEVERY  Set to a time interval to override the default database
+                   check interval of 30 days (720h). The interval understands
+                   "h", "m" and "s" abbreviations for hours minutes and seconds.
+                   Valid values are like "720h", "30s", etc.
+
  GOMAXPROCS        Set the maximum number of CPU cores to use. Defaults to all
                    available CPU cores.
 
@@ -207,9 +220,9 @@ The following are valid values for the STTRACE variable:
 
 // Environment options
 var (
-	noUpgrade       = os.Getenv("STNOUPGRADE") != ""
-	innerProcess    = os.Getenv("STNORESTART") != "" || os.Getenv("STMONITORED") != ""
-	noDefaultFolder = os.Getenv("STNODEFAULTFOLDER") != ""
+	noUpgradeFromEnv = os.Getenv("STNOUPGRADE") != ""
+	innerProcess     = os.Getenv("STNORESTART") != "" || os.Getenv("STMONITORED") != ""
+	noDefaultFolder  = os.Getenv("STNODEFAULTFOLDER") != ""
 )
 
 type RuntimeOptions struct {
@@ -218,6 +231,7 @@ type RuntimeOptions struct {
 	resetDeltaIdxs bool
 	showVersion    bool
 	showPaths      bool
+	showDeviceId   bool
 	doUpgrade      bool
 	doUpgradeCheck bool
 	upgradeTo      string
@@ -252,7 +266,7 @@ func defaultRuntimeOptions() RuntimeOptions {
 	}
 
 	if os.Getenv("STTRACE") != "" {
-		options.logFlags = log.Ltime | log.Ldate | log.Lmicroseconds | log.Lshortfile
+		options.logFlags = logger.DebugFlags
 	}
 
 	if runtime.GOOS != "windows" {
@@ -283,12 +297,13 @@ func parseCommandLineOptions() RuntimeOptions {
 	flag.BoolVar(&options.doUpgradeCheck, "upgrade-check", false, "Check for available upgrade")
 	flag.BoolVar(&options.showVersion, "version", false, "Show version")
 	flag.BoolVar(&options.showPaths, "paths", false, "Show configuration paths")
+	flag.BoolVar(&options.showDeviceId, "device-id", false, "Show the device ID")
 	flag.StringVar(&options.upgradeTo, "upgrade-to", options.upgradeTo, "Force upgrade directly from specified URL")
 	flag.BoolVar(&options.auditEnabled, "audit", false, "Write events to audit file")
 	flag.BoolVar(&options.verbose, "verbose", false, "Print verbose log output")
 	flag.BoolVar(&options.paused, "paused", false, "Start with all devices and folders paused")
 	flag.BoolVar(&options.unpaused, "unpaused", false, "Start with all devices and folders unpaused")
-	flag.StringVar(&options.logFile, "logfile", options.logFile, "Log file name (use \"-\" for stdout)")
+	flag.StringVar(&options.logFile, "logfile", options.logFile, "Log file name (still always logs to stdout). Cannot be used together with -no-restart/STNORESTART environment variable.")
 	flag.StringVar(&options.auditFile, "auditfile", options.auditFile, "Specify audit file (use \"-\" for stdout, \"--\" for stderr)")
 	if runtime.GOOS == "windows" {
 		// Allow user to hide the console window
@@ -308,6 +323,8 @@ func parseCommandLineOptions() RuntimeOptions {
 }
 
 func main() {
+	setBuildMetadata()
+
 	options := parseCommandLineOptions()
 	l.SetFlags(options.logFlags)
 
@@ -334,6 +351,13 @@ func main() {
 
 	if options.confDir != "" {
 		// Not set as default above because the string can be really long.
+		if !filepath.IsAbs(options.confDir) {
+			var err error
+			options.confDir, err = filepath.Abs(options.confDir)
+			if err != nil {
+				l.Fatalln(err)
+			}
+		}
 		baseDirs["config"] = options.confDir
 	}
 
@@ -359,7 +383,18 @@ func main() {
 	}
 
 	if options.showPaths {
-		showPaths()
+		showPaths(options)
+		return
+	}
+
+	if options.showDeviceId {
+		cert, err := tls.LoadX509KeyPair(locations[locCertFile], locations[locKeyFile])
+		if err != nil {
+			l.Fatalln("Error reading device ID:", err)
+		}
+
+		myID = protocol.NewDeviceID(cert.Certificate[0])
+		fmt.Println(myID)
 		return
 	}
 
@@ -401,7 +436,7 @@ func main() {
 		return
 	}
 
-	if options.noRestart {
+	if innerProcess || options.noRestart {
 		syncthingMain(options)
 	} else {
 		monitorMain(options)
@@ -409,7 +444,7 @@ func main() {
 }
 
 func openGUI() {
-	cfg, _ := loadConfig()
+	cfg, _ := loadOrDefaultConfig()
 	if cfg.GUI().Enabled {
 		openURL(cfg.GUI().URL())
 	} else {
@@ -418,7 +453,7 @@ func openGUI() {
 }
 
 func generate(generateDir string) {
-	dir, err := osutil.ExpandTilde(generateDir)
+	dir, err := fs.ExpandTilde(generateDir)
 	if err != nil {
 		l.Fatalln("generate:", err)
 	}
@@ -448,9 +483,7 @@ func generate(generateDir string) {
 		l.Warnln("Config exists; will not overwrite.")
 		return
 	}
-	var myName, _ = os.Hostname()
-	var newCfg = defaultConfig(myName)
-	var cfg = config.Wrap(cfgFile, newCfg)
+	var cfg = defaultConfig(cfgFile)
 	err = cfg.Save()
 	if err != nil {
 		l.Warnln("Failed to save config", err)
@@ -480,9 +513,9 @@ func debugFacilities() string {
 }
 
 func checkUpgrade() upgrade.Release {
-	cfg, _ := loadConfig()
-	releasesURL := cfg.Options().ReleasesURL
-	release, err := upgrade.LatestRelease(releasesURL, Version)
+	cfg, _ := loadOrDefaultConfig()
+	opts := cfg.Options()
+	release, err := upgrade.LatestRelease(opts.ReleasesURL, Version, opts.UpgradeToPreReleases)
 	if err != nil {
 		l.Fatalln("Upgrade:", err)
 	}
@@ -518,7 +551,7 @@ func performUpgrade(release upgrade.Release) {
 }
 
 func upgradeViaRest() error {
-	cfg, _ := loadConfig()
+	cfg, _ := loadOrDefaultConfig()
 	u, err := url.Parse(cfg.GUI().URL())
 	if err != nil {
 		return err
@@ -583,8 +616,8 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	// Event subscription for the API; must start early to catch the early
 	// events. The LocalChangeDetected event might overwhelm the event
 	// receiver in some situations so we will not subscribe to it here.
-	apiSub := events.NewBufferedSubscription(events.Default.Subscribe(events.AllEvents&^events.LocalChangeDetected&^events.RemoteChangeDetected), 1000)
-	diskSub := events.NewBufferedSubscription(events.Default.Subscribe(events.LocalChangeDetected|events.RemoteChangeDetected), 1000)
+	defaultSub := events.NewBufferedSubscription(events.Default.Subscribe(defaultEventMask), eventSubBufferSize)
+	diskSub := events.NewBufferedSubscription(events.Default.Subscribe(diskEventMask), eventSubBufferSize)
 
 	if len(os.Getenv("GOMAXPROCS")) == 0 {
 		runtime.GOMAXPROCS(runtime.NumCPU())
@@ -595,7 +628,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	// report the error if there is one.
 	osutil.MaximizeOpenFileLimit()
 
-	// Ensure that that we have a certificate and key.
+	// Ensure that we have a certificate and key.
 	cert, err := tls.LoadX509KeyPair(locations[locCertFile], locations[locKeyFile])
 	if err != nil {
 		l.Infof("Generating ECDSA key and certificate for %s...", tlsDefaultCommonName)
@@ -611,10 +644,10 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	l.Infoln(LongVersion)
 	l.Infoln("My ID:", myID)
 
+	// Select SHA256 implementation and report. Affected by the
+	// STHASHING environment variable.
 	sha256.SelectAlgo()
 	sha256.Report()
-	perf := cpuBench(3, 150*time.Millisecond)
-	l.Infof("Actual hashing performance is %.02f MB/s", perf)
 
 	// Emit the Starting event, now that we know who we are.
 
@@ -623,7 +656,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		"myID": myID.String(),
 	})
 
-	cfg := loadOrCreateConfig()
+	cfg := loadConfigAtStartup()
 
 	if err := checkShortIDs(cfg); err != nil {
 		l.Fatalln("Short device IDs are in conflict. Unlucky!\n  Regenerate the device ID of one of the following:\n  ", err)
@@ -664,40 +697,37 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		},
 	}
 
-	opts := cfg.Options()
+	if opts := cfg.Options(); opts.WeakHashSelectionMethod == config.WeakHashAuto {
+		perfWithWeakHash := cpuBench(3, 150*time.Millisecond, true)
+		l.Infof("Hashing performance with rolling hash is %.02f MB/s", perfWithWeakHash)
+		perfWithoutWeakHash := cpuBench(3, 150*time.Millisecond, false)
+		l.Infof("Hashing performance without rolling hash is %.02f MB/s", perfWithoutWeakHash)
 
-	if !opts.SymlinksEnabled {
-		symlinks.Supported = false
-	}
-
-	if (opts.MaxRecvKbps > 0 || opts.MaxSendKbps > 0) && !opts.LimitBandwidthInLan {
-		lans, _ = osutil.GetLans()
-		for _, lan := range opts.AlwaysLocalNets {
-			_, ipnet, err := net.ParseCIDR(lan)
-			if err != nil {
-				l.Infoln("Network", lan, "is malformed:", err)
-				continue
-			}
-			lans = append(lans, ipnet)
+		if perfWithoutWeakHash*0.8 > perfWithWeakHash {
+			l.Infof("Rolling hash disabled, as it has an unacceptable performance impact.")
+			weakhash.Enabled = false
+		} else {
+			l.Infof("Rolling hash enabled, as it has an acceptable performance impact.")
+			weakhash.Enabled = true
 		}
-
-		networks := make([]string, len(lans))
-		for i, lan := range lans {
-			networks[i] = lan.String()
-		}
-		l.Infoln("Local networks:", strings.Join(networks, ", "))
+	} else if opts.WeakHashSelectionMethod == config.WeakHashNever {
+		l.Infof("Disabling rolling hash")
+		weakhash.Enabled = false
+	} else if opts.WeakHashSelectionMethod == config.WeakHashAlways {
+		l.Infof("Enabling rolling hash")
+		weakhash.Enabled = true
 	}
 
 	dbFile := locations[locDatabase]
 	ldb, err := db.Open(dbFile)
-
 	if err != nil {
 		l.Fatalln("Cannot open database:", err, "- Is another copy of Syncthing already running?")
 	}
 
 	if runtimeOptions.resetDeltaIdxs {
 		l.Infoln("Reinitializing delta index IDs")
-		ldb.DropDeltaIndexIDs()
+		ldb.DropLocalDeltaIndexIDs()
+		ldb.DropRemoteDeltaIndexIDs()
 	}
 
 	protectedFiles := []string{
@@ -716,19 +746,38 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		}
 	}
 
-	if cfg.RawCopy().OriginalVersion == 15 {
-		// The config version 15->16 migration is about handling ignores and
-		// delta indexes and requires that we drop existing indexes that
-		// have been incorrectly ignore filtered.
-		ldb.DropDeltaIndexIDs()
+	// Grab the previously running version string from the database.
+
+	miscDB := db.NewNamespacedKV(ldb, string(db.KeyTypeMiscData))
+	prevVersion, _ := miscDB.String("prevVersion")
+
+	// Strip away prerelease/beta stuff and just compare the release
+	// numbers. 0.14.44 to 0.14.45-banana is an upgrade, 0.14.45-banana to
+	// 0.14.45-pineapple is not.
+
+	prevParts := strings.Split(prevVersion, "-")
+	curParts := strings.Split(Version, "-")
+	if prevParts[0] != curParts[0] {
+		if prevVersion != "" {
+			l.Infoln("Detected upgrade from", prevVersion, "to", Version)
+		}
+
+		// Drop delta indexes in case we've changed random stuff we
+		// shouldn't have. We will resend our index on next connect.
+		ldb.DropLocalDeltaIndexIDs()
+
+		// Remember the new version.
+		miscDB.PutString("prevVersion", Version)
 	}
 
-	m := model.NewModel(cfg, myID, myDeviceName(cfg), "syncthing", Version, ldb, protectedFiles)
+	// Potential database transitions
+	ldb.UpdateSchema()
 
-	if t := os.Getenv("STDEADLOCKTIMEOUT"); len(t) > 0 {
-		it, err := strconv.Atoi(t)
-		if err == nil {
-			m.StartDeadlockDetector(time.Duration(it) * time.Second)
+	m := model.NewModel(cfg, myID, "syncthing", Version, ldb, protectedFiles)
+
+	if t := os.Getenv("STDEADLOCKTIMEOUT"); t != "" {
+		if secs, _ := strconv.Atoi(t); secs > 0 {
+			m.StartDeadlockDetector(time.Duration(secs) * time.Second)
 		}
 	} else if !IsRelease || IsBeta {
 		m.StartDeadlockDetector(20 * time.Minute)
@@ -743,6 +792,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	// Add and start folders
 	for _, folderCfg := range cfg.Folders() {
 		if folderCfg.Paused {
+			folderCfg.CreateRoot()
 			continue
 		}
 		m.AddFolder(folderCfg)
@@ -758,7 +808,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 
 	// Start connection management
 
-	connectionsService := connections.NewService(cfg, myID, m, tlsCfg, cachedDiscovery, bepProtocolName, tlsDefaultCommonName, lans)
+	connectionsService := connections.NewService(cfg, myID, m, tlsCfg, cachedDiscovery, bepProtocolName, tlsDefaultCommonName)
 	mainService.Add(connectionsService)
 
 	if cfg.Options().GlobalAnnEnabled {
@@ -773,7 +823,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 			// Each global discovery server gets its results cached for five
 			// minutes, and is not asked again for a minute when it's returned
 			// unsuccessfully.
-			cachedDiscovery.Add(gd, 5*time.Minute, time.Minute, globalDiscoveryPriority)
+			cachedDiscovery.Add(gd, 5*time.Minute, time.Minute)
 		}
 	}
 
@@ -783,20 +833,20 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		if err != nil {
 			l.Warnln("IPv4 local discovery:", err)
 		} else {
-			cachedDiscovery.Add(bcd, 0, 0, ipv4LocalDiscoveryPriority)
+			cachedDiscovery.Add(bcd, 0, 0)
 		}
 		// v6 multicasts
 		mcd, err := discover.NewLocal(myID, cfg.Options().LocalAnnMCAddr, connectionsService)
 		if err != nil {
 			l.Warnln("IPv6 local discovery:", err)
 		} else {
-			cachedDiscovery.Add(mcd, 0, 0, ipv6LocalDiscoveryPriority)
+			cachedDiscovery.Add(mcd, 0, 0)
 		}
 	}
 
 	// GUI
 
-	setupGUI(mainService, cfg, m, apiSub, diskSub, cachedDiscovery, connectionsService, errors, systemLog, runtimeOptions)
+	setupGUI(mainService, cfg, m, defaultSub, diskSub, cachedDiscovery, connectionsService, errors, systemLog, runtimeOptions)
 
 	if runtimeOptions.cpuProfile {
 		f, err := os.Create(fmt.Sprintf("cpu-%d.pprof", os.Getpid()))
@@ -812,46 +862,70 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		}
 	}
 
-	if opts.URAccepted > 0 && opts.URAccepted < usageReportVersion {
-		l.Infoln("Anonymous usage report has changed; revoking acceptance")
-		opts.URAccepted = 0
-		opts.URUniqueID = ""
+	// Candidate builds always run with usage reporting.
+
+	if opts := cfg.Options(); IsCandidate {
+		l.Infoln("Anonymous usage reporting is always enabled for candidate releases.")
+		opts.URAccepted = usageReportVersion
 		cfg.SetOptions(opts)
-	}
-	if opts.URAccepted >= usageReportVersion {
-		if opts.URUniqueID == "" {
-			// Previously the ID was generated from the node ID. We now need
-			// to generate a new one.
-			opts.URUniqueID = rand.String(8)
-			cfg.SetOptions(opts)
-			cfg.Save()
-		}
+		cfg.Save()
+		// Unique ID will be set and config saved below if necessary.
 	}
 
-	// The usageReportingManager registers itself to listen to configuration
-	// changes, and there's nothing more we need to tell it from the outside.
-	// Hence we don't keep the returned pointer.
-	newUsageReportingManager(cfg, m)
+	if opts := cfg.Options(); opts.URUniqueID == "" {
+		opts.URUniqueID = rand.String(8)
+		cfg.SetOptions(opts)
+		cfg.Save()
+	}
 
-	if opts.RestartOnWakeup {
+	usageReportingSvc := newUsageReportingService(cfg, m, connectionsService)
+	mainService.Add(usageReportingSvc)
+
+	if opts := cfg.Options(); opts.RestartOnWakeup {
 		go standbyMonitor()
 	}
 
-	if opts.AutoUpgradeIntervalH > 0 {
-		if noUpgrade {
+	// Candidate builds should auto upgrade. Make sure the option is set,
+	// unless we are in a build where it's disabled or the STNOUPGRADE
+	// environment variable is set.
+
+	if IsCandidate && !upgrade.DisabledByCompilation && !noUpgradeFromEnv {
+		l.Infoln("Automatic upgrade is always enabled for candidate releases.")
+		if opts := cfg.Options(); opts.AutoUpgradeIntervalH == 0 || opts.AutoUpgradeIntervalH > 24 {
+			opts.AutoUpgradeIntervalH = 12
+			// Set the option into the config as well, as the auto upgrade
+			// loop expects to read a valid interval from there.
+			cfg.SetOptions(opts)
+			cfg.Save()
+		}
+		// We don't tweak the user's choice of upgrading to pre-releases or
+		// not, as otherwise they cannot step off the candidate channel.
+	}
+
+	if opts := cfg.Options(); opts.AutoUpgradeIntervalH > 0 {
+		if noUpgradeFromEnv {
 			l.Infof("No automatic upgrades; STNOUPGRADE environment variable defined.")
 		} else {
 			go autoUpgrade(cfg)
 		}
 	}
 
+	if isSuperUser() {
+		l.Warnln("Syncthing should not run as a privileged or system user. Please consider using a normal user account.")
+	}
+
 	events.Default.Log(events.StartupComplete, map[string]string{
 		"myID": myID.String(),
 	})
-	go generatePingEvents()
 
 	cleanConfigDirectory()
 
+	if cfg.Options().SetLowPriority {
+		if err := osutil.SetLowPriority(); err != nil {
+			l.Warnln("Failed to lower process priority:", err)
+		}
+	}
+
 	code := <-stop
 
 	mainService.Stop()
@@ -865,15 +939,6 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	os.Exit(code)
 }
 
-func myDeviceName(cfg *config.Wrapper) string {
-	devices := cfg.Devices()
-	myName := devices[myID].Name
-	if myName == "" {
-		myName, _ = os.Hostname()
-	}
-	return myName
-}
-
 func setupSignalHandling() {
 	// Exit cleanly with "restarting" code on SIGHUP.
 
@@ -896,26 +961,28 @@ func setupSignalHandling() {
 	}()
 }
 
-func loadConfig() (*config.Wrapper, error) {
+func loadOrDefaultConfig() (*config.Wrapper, error) {
 	cfgFile := locations[locConfigFile]
 	cfg, err := config.Load(cfgFile, myID)
 
 	if err != nil {
-		myName, _ := os.Hostname()
-		newCfg := defaultConfig(myName)
-		cfg = config.Wrap(cfgFile, newCfg)
+		cfg = defaultConfig(cfgFile)
 	}
 
 	return cfg, err
 }
 
-func loadOrCreateConfig() *config.Wrapper {
-	cfg, err := loadConfig()
+func loadConfigAtStartup() *config.Wrapper {
+	cfgFile := locations[locConfigFile]
+	cfg, err := config.Load(cfgFile, myID)
 	if os.IsNotExist(err) {
+		cfg = defaultConfig(cfgFile)
 		cfg.Save()
-		l.Infof("Defaults saved. Edit %s to taste or use the GUI\n", cfg.ConfigPath())
+		l.Infof("Default config saved. Edit %s to taste or use the GUI\n", cfg.ConfigPath())
+	} else if err == io.EOF {
+		l.Fatalln("Failed to load config: unexpected end of file. Truncated or empty configuration?")
 	} else if err != nil {
-		l.Fatalln("Config:", err)
+		l.Fatalln("Failed to load config:", err)
 	}
 
 	if cfg.RawCopy().OriginalVersion != config.CurrentVersion {
@@ -992,7 +1059,7 @@ func startAuditing(mainService *suture.Supervisor, auditFile string) {
 	l.Infoln("Audit log in", auditDest)
 }
 
-func setupGUI(mainService *suture.Supervisor, cfg *config.Wrapper, m *model.Model, apiSub events.BufferedSubscription, diskSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService *connections.Service, errors, systemLog logger.Recorder, runtimeOptions RuntimeOptions) {
+func setupGUI(mainService *suture.Supervisor, cfg *config.Wrapper, m *model.Model, defaultSub, diskSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService *connections.Service, errors, systemLog logger.Recorder, runtimeOptions RuntimeOptions) {
 	guiCfg := cfg.GUI()
 
 	if !guiCfg.Enabled {
@@ -1003,30 +1070,29 @@ func setupGUI(mainService *suture.Supervisor, cfg *config.Wrapper, m *model.Mode
 		l.Warnln("Insecure admin access is enabled.")
 	}
 
-	api := newAPIService(myID, cfg, locations[locHTTPSCertFile], locations[locHTTPSKeyFile], runtimeOptions.assetDir, m, apiSub, diskSub, discoverer, connectionsService, errors, systemLog)
+	cpu := newCPUService()
+	mainService.Add(cpu)
+
+	api := newAPIService(myID, cfg, locations[locHTTPSCertFile], locations[locHTTPSKeyFile], runtimeOptions.assetDir, m, defaultSub, diskSub, discoverer, connectionsService, errors, systemLog, cpu)
 	cfg.Subscribe(api)
 	mainService.Add(api)
 
 	if cfg.Options().StartBrowser && !runtimeOptions.noBrowser && !runtimeOptions.stRestarting {
 		// Can potentially block if the utility we are invoking doesn't
-		// fork, and just execs, hence keep it in it's own routine.
+		// fork, and just execs, hence keep it in its own routine.
 		<-api.startedOnce
 		go openURL(guiCfg.URL())
 	}
 }
 
-func defaultConfig(myName string) config.Configuration {
+func defaultConfig(cfgFile string) *config.Wrapper {
+	myName, _ := os.Hostname()
+
 	var defaultFolder config.FolderConfiguration
 
 	if !noDefaultFolder {
 		l.Infoln("Default folder created and/or linked to new config")
-		defaultFolder = config.NewFolderConfiguration("default", locations[locDefFolder])
-		defaultFolder.Label = "Default Folder"
-		defaultFolder.RescanIntervalS = 60
-		defaultFolder.MinDiskFreePct = 1
-		defaultFolder.Devices = []config.FolderDeviceConfiguration{{DeviceID: myID}}
-		defaultFolder.AutoNormalize = true
-		defaultFolder.MaxConflicts = -1
+		defaultFolder = config.NewFolderConfiguration(myID, "default", "Default Folder", fs.FilesystemTypeBasic, locations[locDefFolder])
 	} else {
 		l.Infoln("We will skip creation of a default folder on first start since the proper envvar is set")
 	}
@@ -1059,14 +1125,7 @@ func defaultConfig(myName string) config.Configuration {
 		}
 	}
 
-	return newCfg
-}
-
-func generatePingEvents() {
-	for {
-		time.Sleep(pingEventInterval)
-		events.Default.Log(events.Ping, nil)
-	}
+	return config.Wrap(cfgFile, newCfg)
 }
 
 func resetDB() error {
@@ -1083,19 +1142,20 @@ func shutdown() {
 	stop <- exitSuccess
 }
 
-func ensureDir(dir string, mode os.FileMode) {
-	err := osutil.MkdirAll(dir, mode)
+func ensureDir(dir string, mode fs.FileMode) {
+	fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
+	err := fs.MkdirAll(".", mode)
 	if err != nil {
 		l.Fatalln(err)
 	}
 
-	if fi, err := os.Stat(dir); err == nil {
+	if fi, err := fs.Stat("."); err == nil {
 		// Apprently the stat may fail even though the mkdirall passed. If it
 		// does, we'll just assume things are in order and let other things
 		// fail (like loading or creating the config...).
 		currentMode := fi.Mode() & 0777
 		if currentMode != mode {
-			err := os.Chmod(dir, mode)
+			err := fs.Chmod(".", mode)
 			// This can fail on crappy filesystems, nothing we can do about it.
 			if err != nil {
 				l.Warnln(err)
@@ -1159,7 +1219,15 @@ func autoUpgrade(cfg *config.Wrapper) {
 		case <-timer.C:
 		}
 
-		rel, err := upgrade.LatestRelease(cfg.Options().ReleasesURL, Version)
+		opts := cfg.Options()
+		checkInterval := time.Duration(opts.AutoUpgradeIntervalH) * time.Hour
+		if checkInterval < time.Hour {
+			// We shouldn't be here if AutoUpgradeIntervalH < 1, but for
+			// safety's sake.
+			checkInterval = time.Hour
+		}
+
+		rel, err := upgrade.LatestRelease(opts.ReleasesURL, Version, opts.UpgradeToPreReleases)
 		if err == upgrade.ErrUpgradeUnsupported {
 			events.Default.Unsubscribe(sub)
 			return
@@ -1168,13 +1236,13 @@ func autoUpgrade(cfg *config.Wrapper) {
 			// Don't complain too loudly here; we might simply not have
 			// internet connectivity, or the upgrade server might be down.
 			l.Infoln("Automatic upgrade:", err)
-			timer.Reset(time.Duration(cfg.Options().AutoUpgradeIntervalH) * time.Hour)
+			timer.Reset(checkInterval)
 			continue
 		}
 
 		if upgrade.CompareVersions(rel.Tag, Version) != upgrade.Newer {
 			// Skip equal, older or majorly newer (incompatible) versions
-			timer.Reset(time.Duration(cfg.Options().AutoUpgradeIntervalH) * time.Hour)
+			timer.Reset(checkInterval)
 			continue
 		}
 
@@ -1182,7 +1250,7 @@ func autoUpgrade(cfg *config.Wrapper) {
 		err = upgrade.To(rel)
 		if err != nil {
 			l.Warnln("Automatic upgrade:", err)
-			timer.Reset(time.Duration(cfg.Options().AutoUpgradeIntervalH) * time.Hour)
+			timer.Reset(checkInterval)
 			continue
 		}
 		events.Default.Unsubscribe(sub)
@@ -1210,22 +1278,22 @@ func cleanConfigDirectory() {
 	}
 
 	for pat, dur := range patterns {
-		pat = filepath.Join(baseDirs["config"], pat)
-		files, err := osutil.Glob(pat)
+		fs := fs.NewFilesystem(fs.FilesystemTypeBasic, baseDirs["config"])
+		files, err := fs.Glob(pat)
 		if err != nil {
 			l.Infoln("Cleaning:", err)
 			continue
 		}
 
 		for _, file := range files {
-			info, err := osutil.Lstat(file)
+			info, err := fs.Lstat(file)
 			if err != nil {
 				l.Infoln("Cleaning:", err)
 				continue
 			}
 
 			if time.Since(info.ModTime()) > dur {
-				if err = os.RemoveAll(file); err != nil {
+				if err = fs.RemoveAll(file); err != nil {
 					l.Infoln("Cleaning:", err)
 				} else {
 					l.Infoln("Cleaned away old file", filepath.Base(file))
@@ -1250,13 +1318,13 @@ func checkShortIDs(cfg *config.Wrapper) error {
 	return nil
 }
 
-func showPaths() {
+func showPaths(options RuntimeOptions) {
 	fmt.Printf("Configuration file:\n\t%s\n\n", locations[locConfigFile])
 	fmt.Printf("Database directory:\n\t%s\n\n", locations[locDatabase])
 	fmt.Printf("Device private key & certificate files:\n\t%s\n\t%s\n\n", locations[locKeyFile], locations[locCertFile])
 	fmt.Printf("HTTPS private key & certificate files:\n\t%s\n\t%s\n\n", locations[locHTTPSKeyFile], locations[locHTTPSCertFile])
-	fmt.Printf("Log file:\n\t%s\n\n", locations[locLogFile])
-	fmt.Printf("GUI override directory:\n\t%s\n\n", locations[locGUIAssets])
+	fmt.Printf("Log file:\n\t%s\n\n", options.logFile)
+	fmt.Printf("GUI override directory:\n\t%s\n\n", options.assetDir)
 	fmt.Printf("Default sync folder directory:\n\t%s\n\n", locations[locDefFolder])
 }
 
@@ -1268,7 +1336,7 @@ func setPauseState(cfg *config.Wrapper, paused bool) {
 	for i := range raw.Folders {
 		raw.Folders[i].Paused = paused
 	}
-	if err := cfg.Replace(raw); err != nil {
+	if _, err := cfg.Replace(raw); err != nil {
 		l.Fatalln("Cannot adjust paused state:", err)
 	}
 }

cmd/syncthing/main_test.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/memsize_darwin.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/memsize_linux.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/memsize_netbsd.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/memsize_solaris.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 // +build solaris
 

cmd/syncthing/memsize_unimpl.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 // +build freebsd openbsd dragonfly
 

cmd/syncthing/memsize_windows.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/mocked_config_test.go

@@ -2,13 +2,14 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/util"
 )
 
 type mockedConfig struct {
@@ -24,15 +25,17 @@ func (c *mockedConfig) ListenAddresses() []string {
 }
 
 func (c *mockedConfig) RawCopy() config.Configuration {
-	return config.Configuration{}
+	cfg := config.Configuration{}
+	util.SetDefaults(&cfg.Options)
+	return cfg
 }
 
 func (c *mockedConfig) Options() config.OptionsConfiguration {
 	return config.OptionsConfiguration{}
 }
 
-func (c *mockedConfig) Replace(cfg config.Configuration) error {
-	return nil
+func (c *mockedConfig) Replace(cfg config.Configuration) (config.Waiter, error) {
+	return nil, nil
 }
 
 func (c *mockedConfig) Subscribe(cm config.Committer) {}
@@ -45,8 +48,12 @@ func (c *mockedConfig) Devices() map[protocol.DeviceID]config.DeviceConfiguratio
 	return nil
 }
 
-func (c *mockedConfig) SetDevice(config.DeviceConfiguration) error {
-	return nil
+func (c *mockedConfig) SetDevice(config.DeviceConfiguration) (config.Waiter, error) {
+	return nil, nil
+}
+
+func (c *mockedConfig) SetDevices([]config.DeviceConfiguration) (config.Waiter, error) {
+	return nil, nil
 }
 
 func (c *mockedConfig) Save() error {

cmd/syncthing/mocked_connections_test.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -11,3 +11,7 @@ type mockedConnections struct{}
 func (m *mockedConnections) Status() map[string]interface{} {
 	return nil
 }
+
+func (m *mockedConnections) NATType() string {
+	return ""
+}

cmd/syncthing/mocked_cpuusage_test.go

@@ -0,0 +1,13 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+type mockedCPUService struct{}
+
+func (*mockedCPUService) Rate() float64 {
+	return 42
+}

cmd/syncthing/mocked_discovery_test.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -44,7 +44,7 @@ func (m *mockedCachingMux) Cache() map[protocol.DeviceID]discover.CacheEntry {
 
 // from events.CachingMux
 
-func (m *mockedCachingMux) Add(finder discover.Finder, cacheTime, negCacheTime time.Duration, priority int) {
+func (m *mockedCachingMux) Add(finder discover.Finder, cacheTime, negCacheTime time.Duration) {
 }
 
 func (m *mockedCachingMux) ChildErrors() map[string]error {

cmd/syncthing/mocked_events_test.go

@@ -2,14 +2,18 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
-import "github.com/syncthing/syncthing/lib/events"
+import (
+	"time"
+
+	"github.com/syncthing/syncthing/lib/events"
+)
 
 type mockedEventSub struct{}
 
-func (s *mockedEventSub) Since(id int, into []events.Event) []events.Event {
+func (s *mockedEventSub) Since(id int, into []events.Event, timeout time.Duration) []events.Event {
 	select {}
 }

cmd/syncthing/mocked_logger_test.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/mocked_model_test.go

@@ -2,17 +2,19 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
 	"time"
 
+	"github.com/syncthing/syncthing/lib/connections"
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/stats"
+	"github.com/syncthing/syncthing/lib/versioner"
 )
 
 type mockedModel struct{}
@@ -27,8 +29,12 @@ func (m *mockedModel) Completion(device protocol.DeviceID, folder string) model.
 
 func (m *mockedModel) Override(folder string) {}
 
-func (m *mockedModel) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated, int) {
-	return nil, nil, nil, 0
+func (m *mockedModel) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) {
+	return nil, nil, nil
+}
+
+func (m *mockedModel) RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) {
+	return nil, nil
 }
 
 func (m *mockedModel) NeedSize(folder string) db.Counts {
@@ -70,6 +76,14 @@ func (m *mockedModel) SetIgnores(folder string, content []string) error {
 	return nil
 }
 
+func (m *mockedModel) GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error) {
+	return nil, nil
+}
+
+func (m *mockedModel) RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error) {
+	return nil, nil
+}
+
 func (m *mockedModel) PauseDevice(device protocol.DeviceID) {
 }
 
@@ -91,8 +105,8 @@ func (m *mockedModel) ScanFolderSubdirs(folder string, subs []string) error {
 
 func (m *mockedModel) BringToFront(folder, file string) {}
 
-func (m *mockedModel) ConnectedTo(deviceID protocol.DeviceID) bool {
-	return false
+func (m *mockedModel) Connection(deviceID protocol.DeviceID) (connections.Connection, bool) {
+	return nil, false
 }
 
 func (m *mockedModel) GlobalSize(folder string) db.Counts {
@@ -114,3 +128,15 @@ func (m *mockedModel) RemoteSequence(folder string) (int64, bool) {
 func (m *mockedModel) State(folder string) (string, time.Time, error) {
 	return "", time.Time{}, nil
 }
+
+func (m *mockedModel) UsageReportingStats(version int, preview bool) map[string]interface{} {
+	return nil
+}
+
+func (m *mockedModel) PullErrors(folder string) ([]model.FileError, error) {
+	return nil, nil
+}
+
+func (m *mockedModel) WatchError(folder string) error {
+	return nil
+}

cmd/syncthing/monitor.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -35,8 +35,6 @@ const (
 )
 
 func monitorMain(runtimeOptions RuntimeOptions) {
-	os.Setenv("STNORESTART", "yes")
-	os.Setenv("STMONITORED", "yes")
 	l.SetPrefix("[monitor] ")
 
 	var dst io.Writer = os.Stdout
@@ -70,6 +68,8 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 	sigHup := syscall.Signal(1)
 	signal.Notify(restartSign, sigHup)
 
+	childEnv := childEnv()
+	first := true
 	for {
 		if t := time.Since(restarts[0]); t < loopThreshold {
 			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
@@ -80,6 +80,7 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 		restarts[len(restarts)-1] = time.Now()
 
 		cmd := exec.Command(args[0], args[1:]...)
+		cmd.Env = childEnv
 
 		stderr, err := cmd.StderrPipe()
 		if err != nil {
@@ -97,10 +98,6 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 			l.Fatalln(err)
 		}
 
-		// Let the next child process know that this is not the first time
-		// it's starting up.
-		os.Setenv("STRESTART", "yes")
-
 		stdoutMut.Lock()
 		stdoutFirstLines = make([]string, 0, 10)
 		stdoutLastLines = make([]string, 0, 50)
@@ -150,7 +147,6 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 						// Restart the monitor process to release the .old
 						// binary as part of the upgrade process.
 						l.Infoln("Restarting monitor...")
-						os.Setenv("STNORESTART", "")
 						if err = restartMonitor(args); err != nil {
 							l.Warnln("Restart:", err)
 						}
@@ -162,6 +158,13 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 
 		l.Infoln("Syncthing exited:", err)
 		time.Sleep(1 * time.Second)
+
+		if first {
+			// Let the next child process know that this is not the first time
+			// it's starting up.
+			childEnv = append(childEnv, "STRESTART=yes")
+			first = false
+		}
 	}
 }
 
@@ -410,3 +413,19 @@ func (f *autoclosedFile) closerLoop() {
 		}
 	}
 }
+
+// Returns the desired child environment, properly filtered and added to.
+func childEnv() []string {
+	var env []string
+	for _, str := range os.Environ() {
+		if strings.HasPrefix("STNORESTART=", str) {
+			continue
+		}
+		if strings.HasPrefix("STMONITORED=", str) {
+			continue
+		}
+		env = append(env, str)
+	}
+	env = append(env, "STMONITORED=yes")
+	return env
+}

cmd/syncthing/monitor_test.go

@@ -0,0 +1,88 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestAutoClosedFile(t *testing.T) {
+	os.RemoveAll("_autoclose")
+	defer os.RemoveAll("_autoclose")
+	os.Mkdir("_autoclose", 0755)
+	file := filepath.FromSlash("_autoclose/tmp")
+	data := []byte("hello, world\n")
+
+	// An autoclosed file that closes very quickly
+	ac := newAutoclosedFile(file, time.Millisecond, time.Millisecond)
+
+	// Write some data.
+	if _, err := ac.Write(data); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for it to close
+	start := time.Now()
+	for {
+		time.Sleep(time.Millisecond)
+		ac.mut.Lock()
+		fd := ac.fd
+		ac.mut.Unlock()
+		if fd == nil {
+			break
+		}
+		if time.Since(start) > time.Second {
+			t.Fatal("File should have been closed after first write")
+		}
+	}
+
+	// Write more data, which should be an append.
+	if _, err := ac.Write(data); err != nil {
+		t.Fatal(err)
+	}
+
+	// Close.
+	if err := ac.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// The file should have both writes in it.
+	bs, err := ioutil.ReadFile(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(bs) != 2*len(data) {
+		t.Fatalf("Writes failed, expected %d bytes, not %d", 2*len(data), len(bs))
+	}
+
+	// Open the file again.
+	ac = newAutoclosedFile(file, time.Second, time.Second)
+
+	// Write something
+	if _, err := ac.Write(data); err != nil {
+		t.Fatal(err)
+	}
+
+	// It should now contain only one write, because the first open
+	// should be a truncate.
+	bs, err = ioutil.ReadFile(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(bs) != len(data) {
+		t.Fatalf("Write failed, expected %d bytes, not %d", len(data), len(bs))
+	}
+
+	// Close.
+	if err := ac.Close(); err != nil {
+		t.Fatal(err)
+	}
+}

cmd/syncthing/openurl_unix.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 // +build !windows
 

cmd/syncthing/openurl_windows.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 // +build windows
 

cmd/syncthing/perfstats_unix.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 // +build !solaris,!windows
 

cmd/syncthing/summaryservice.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -169,7 +169,7 @@ func (c *folderSummaryService) foldersToHandle() []string {
 	c.lastEventReqMut.Lock()
 	last := c.lastEventReq
 	c.lastEventReqMut.Unlock()
-	if time.Since(last) > pingEventInterval {
+	if time.Since(last) > defaultEventTimeout {
 		return nil
 	}
 
@@ -187,7 +187,10 @@ func (c *folderSummaryService) foldersToHandle() []string {
 func (c *folderSummaryService) sendSummary(folder string) {
 	// The folder summary contains how many bytes, files etc
 	// are in the folder and how in sync we are.
-	data := folderSummary(c.cfg, c.model, folder)
+	data, err := folderSummary(c.cfg, c.model, folder)
+	if err != nil {
+		return
+	}
 	events.Default.Log(events.FolderSummary, map[string]interface{}{
 		"folder":  folder,
 		"summary": data,
@@ -198,21 +201,17 @@ func (c *folderSummaryService) sendSummary(folder string) {
 			// We already know about ourselves.
 			continue
 		}
-		if !c.model.ConnectedTo(devCfg.DeviceID) {
+		if _, ok := c.model.Connection(devCfg.DeviceID); !ok {
 			// We're not interested in disconnected devices.
 			continue
 		}
 
 		// Get completion percentage of this folder for the
 		// remote device.
-		comp := c.model.Completion(devCfg.DeviceID, folder)
-		events.Default.Log(events.FolderCompletion, map[string]interface{}{
-			"folder":      folder,
-			"device":      devCfg.DeviceID.String(),
-			"completion":  comp.CompletionPct,
-			"needBytes":   comp.NeedBytes,
-			"globalBytes": comp.GlobalBytes,
-		})
+		comp := jsonCompletion(c.model.Completion(devCfg.DeviceID, folder))
+		comp["folder"] = folder
+		comp["device"] = devCfg.DeviceID.String()
+		events.Default.Log(events.FolderCompletion, comp)
 	}
 }
 

cmd/syncthing/superuser_unix.go

@@ -0,0 +1,17 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// +build !windows
+
+package main
+
+import (
+	"os"
+)
+
+func isSuperUser() bool {
+	return os.Geteuid() == 0
+}

cmd/syncthing/superuser_windows.go

@@ -0,0 +1,41 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import "syscall"
+
+// https://msdn.microsoft.com/en-us/library/windows/desktop/aa379649(v=vs.85).aspx
+const securityLocalSystemRID = "S-1-5-18"
+
+func isSuperUser() bool {
+	tok, err := syscall.OpenCurrentProcessToken()
+	if err != nil {
+		l.Debugln("OpenCurrentProcessToken:", err)
+		return false
+	}
+	defer tok.Close()
+
+	user, err := tok.GetTokenUser()
+	if err != nil {
+		l.Debugln("GetTokenUser:", err)
+		return false
+	}
+
+	if user.User.Sid == nil {
+		l.Debugln("sid is nil")
+		return false
+	}
+
+	sid, err := user.User.Sid.String()
+	if err != nil {
+		l.Debugln("Sid.String():", err)
+		return false
+	}
+
+	l.Debugf("SID: %q", sid)
+	return sid == securityLocalSystemRID
+}

cmd/syncthing/traceback.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 //+build go1.7