cmd/*, lib/build: Set correct LongVersion (fixes #5993) (#5997)

The relay and discosrv didn't use the new lib/build package, now they
do. Conversely the lib/build package wasn't aware there might be other
users and hard coded the program name - now it's set by the build
script

.codecov.yml

@@ -0,0 +1,5 @@
+coverage:
+  range: "40...100"
+
+ignore:
+  - "**.pb.go"

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+/AUTHORS    @calmh
+/*.md       @calmh

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,42 @@
+### DO NOT REPORT SECURITY ISSUES IN THIS ISSUE TRACKER
+
+Instead, contact security@syncthing.net directly - see
+https://syncthing.net/security.html for more information.
+
+### DO NOT POST SUPPORT REQUESTS OR GENERAL QUESTIONS IN THIS ISSUE TRACKER
+
+Please use the forum at https://forum.syncthing.net/ where a large number of
+helpful people hang out. This issue tracker is for reporting bugs or feature
+requests directly to the developers. Worst case you might get a short
+"that's a bug, please report it on GitHub" response on the forum, in which
+case we thank you for your patience and following our advice. :)
+
+### Please use the correct issue tracker
+
+If your problem relates to a Syncthing wrapper or [sub-project](https://github.com/syncthing) such as [Syncthing for Android](https://github.com/syncthing/syncthing-android/issues), [SyncTrayzor](https://github.com/canton7/synctrayzor) or the [documentation](https://github.com/syncthing/docs/issues), please use their respective issue trackers.
+
+### Does your log mention database corruption?
+
+If your Syncthing log reports panics because of database corruption it is most likely a fault with your system's storage or memory. Affected log entries will contain lines starting with `panic: leveldb`. You will need to delete the index database to clear this, by running `syncthing -reset-database`.
+
+### Please do post actual bug reports and feature requests.
+
+If your issue is a bug report, replace this boilerplate with a description
+of the problem, being sure to include at least:
+
+ - what happened,
+ - what you expected to happen instead, and
+ - any steps to reproduce the problem.
+
+Also fill out the version information below and add log output or
+screenshots as appropriate.
+
+If your issue is a feature request, simply replace this template text in
+its entirety.
+
+### Version Information
+
+Syncthing Version: v0.x.y
+OS Version: Windows 7 / Ubuntu 14.04 / ...
+Browser Version: (if applicable, for GUI issues)
+

.github/PULL_REQUEST_TEMPLATE.md

@@ -20,13 +20,8 @@ If this is a user visible change (including API and protocol changes), add a lin
 to the corresponding pull request on https://github.com/syncthing/docs or describe
 the documentation changes necessary.
 
-### Authorship
+## Authorship
+
+Your name and email will be added automatically to the AUTHORS file
+based on the commit metadata.
 
-Every author of a code contribution (Go, Javascript, HTML, CSS etc, with the
-possible exception of minor typo corrections and similar) is recorded in the
-AUTHORS and NICKS files and the in-GUI credits. If this is your first
-contribution, a maintainer will add you properly before accepting the
-contribution. You need not do so yourself or worry about the fact that the
-"authors" automated test fails. However, if your name (such as you want it
-presented in the credits) is not visible on your Github profile or in your
-commit messages, please assist by providing it here.

.github/SECURITY.md

@@ -0,0 +1,48 @@
+## Reporting a Vulnerability
+
+If you believe that you've found a Syncthing-related security vulnerability, please report it by sending email to the address security@syncthing.net. The PGP key for security@syncthing.net (B683AD7B76CAB013) below can be used to send encrypted mail or to verify responses received from that address.
+
+```
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQENBFShFlIBCACsW346HYskhKhxrdZMjyU5Ntsjvg6/ogqINDPoL10/oaIP0G+t
+7zzC0K5Cq29ix43kNNLKTJNyPkdTeaJEcqslMUt6tovjHwoKJ073GP0W3KsNvBRg
+ffCZOAexGfOsBSL9KHaYGK67Py3TFgtN1H/EmboU1arrLfAMrmqOip++EGqOxjse
+gH0qk7Mk1TqEC5Xh3NGE7r1UobAlqdUv5E3v7U11NhAdP1zu/XZ/zvP5mwVQJMLv
+iZyeWGliNI8nEeRjYw+S85f4gq7H2mgoeNBN4WwwK1hhz9qpvCsgPW3XqlExTPI4
+1vM4PxpiFIuF0zuy2OwsmjrpTCZeBscr4Tj5ABEBAAG0K1N5bmN0aGluZyBTZWN1
+cml0eSA8c2VjdXJpdHlAc3luY3RoaW5nLm5ldD6JATgEEwECACIFAlShFlICGwMG
+CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELaDrXt2yrATB1UH/jnnIa6DekCA
+2V36N/2+pFSNLVWeoZQxZ9ne9S7WSubaoK4oCWiuChPSAy5hagnKnNA3a9wrz5iN
+0hgCA88NnTU18biZW6HX8xWEd3dnY3fX1sG0XdHuKlFria8ByfcrbShf/CttXkEd
+Y5qPHH9aLIMtksBS1MIsRYrOCHiLNYFCKlbjDDCGT3tuk/yaU7aBAFVPIag54afC
+zZAIvTIgRruLWkNT/sSEJx9ekhJzO/+pXNbSSHwhoj5OJh7sQjZWwPzNazelk48R
++ku8VpB5Wxgk2MnJPf1RxF+7saBAaeAVZsJcPN+Jp7u9LCFelIRn1ISsHbhLhyqL
+cPXqllltLCKJAhwEEAECAAYFAlShFlsACgkQSfWuwLzlJMcEpA//VvZYHGZgZMM0
+bBkYnjManYnJkHSQ2pHsjquSFH+fbfq2FxxTk/nQ/IAvBzt8NDTT3ylPOmsl4A8q
+r8BxsRNENhU7zDQrKC5NtYUrzXhPGo8qfDwkeLyqd2msUvHn7EH3PNgiN2QaFWxE
+21FqoXIpERKJRgRtv1SYZoMyNGjmT2hta1ZbwEfrPJnzjYhneoUGsRApG7p+uPqe
+4LRpbG3Fa2vBm/UWUrOe+6jPzvMokjIJbdK0IjXamFAzwYW5fSZaFa94mR6L5f5m
+X8Dhp66mBRTx7qk6ldEqptvaYGqihaWP1xbDaApQsGHQujtcBYGp+edlkabuW5h7
+stl/7QTFkEPqHT4ybxEX0rLoFUGq56WUlKp27z40keStaXMgfrsxJtkz66Xs8vU4
+7qZcLAcPsin+y0toqavtwtM/L7uCMu1yhbFRGJ+JL6saJAqzwS8l7r6E2R7OnVj1
+BdASgxx1TgzW6ZFW5p1Iy6mtpkBypsnp8s3UcP3GFRnQe9gi1EXHjzuZAUGafe4Q
+juvJ8t8xIcQMFuAylNIVyXvIWJoehqsVY9EBgVtE4y1SRh8XTT3Tddn8ab5fl7uh
+HWAY8cRlv6WIOhK8w8oroiYx0SID8jMeEwJBS4DL7qWtJMkDo8ZEJiB+Pkd963MX
+05QXt33AvJJ9PmbGCHkcH7198tCmA+e5AQ0EVKEWUgEIAPGczHpa6NdxY0pm+tIp
+btiA6gdPE70pJYgJTKX7siCQ2w770H3CBSKmqEXadr7WnfIgUYIDaSxadeGzB/Mn
+3SHCYRCqbA7mwu2k4wNNvCEM0xZqFAvaDJ4avlZ5oiMT8IFHKsjC77nkhmfXaIGt
+hn10H2MFADjvJYul7vR+Ghg1wGhTGWo2u7VVj9BI+AfvnWaouFI0cx2KNWEI/Ocj
+z6jk8nmC3yOEFQECM/hF4lkAOv9CQUa8UcvAr31trzawmV1iSsKjmVZgqd0N4T8f
+hUikqUPZGNCRcqEUffTzggIyGPbedFnZw9Di7o1xByxyTrZycemAVqaVGF+9nFLG
+pccAEQEAAYkBHwQYAQIACQUCVKEWUgIbDAAKCRC2g617dsqwEzrjB/9q0T8A4XUE
+p0g6xq86jhmh4jlEedxrfXUL3R6ejFtuKMThulxEP0xiQ7xLzBhOnvyxLCsVbjp8
+0JtJTVCq44UzUiIBuVNRoYG29uXuTUL2UtI27VhjFxMzLDwZL97tbGR6lzdM/+U5
+9PC+PIvS0yz13z2t3/x3KUOnBgxZnpy9h4AdKwjrNVtnQdGDXSKlJBLb57TFcS94
+f3roZ5Gpw3AWYSmSSiZWbhks2UNzYSQ84LAKlV1NkktO+qRc/pUqr6IxMjOKc7XP
+e8u1Nst6fN3GNqZOV+jUYs/fqrJgp1TUWjNTuf22Rl0Idz7XLPJKYFh9W7T/4MbU
+M7Q8GZuww1rk
+=No/v
+-----END PGP PUBLIC KEY BLOCK-----
+```

.gitignore

@@ -1,11 +1,11 @@
-syncthing
-!gui/syncthing
-!debian/syncthing
-!Godeps/_workspace/src/github.com/syncthing
+/syncthing
+/stdiscosrv
 syncthing.exe
+stdiscosrv.exe
 *.tar.gz
 *.zip
 *.asc
+*.deb
 .jshintrc
 coverage.out
 files/pidx
@@ -16,3 +16,11 @@ syncthing.sig
 RELEASE
 deb
 lib/auto/gui.files.go
+snapcraft.yaml
+prime/
+snap/
+parts/
+stage/
+*.snap
+*.bz2
+/repos

.golangci.yml

@@ -0,0 +1,21 @@
+linters-settings:
+  maligned:
+    suggest-new: true
+
+linters:
+  enable-all: true
+  disable:
+    - goimports
+    - depguard
+    - lll
+    - gochecknoinits
+    - gochecknoglobals
+    - gofmt
+    - scopelint
+
+service:
+  golangci-lint-version: 1.17.x
+  prepare:
+    - rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
+    - GO111MODULE=on go mod vendor
+    - go run build.go assets

.mailmap

@@ -1 +0,0 @@
-NICKS

AUTHORS

@@ -1,90 +1,208 @@
 # This is the official list of Syncthing authors for copyright purposes.
+#
+# THIS FILE IS MOSTLY AUTO GENERATED. IF YOU'VE MADE A COMMIT TO THE
+# REPOSITORY YOU WILL BE ADDED HERE AUTOMATICALLY WITHOUT THE NEED FOR
+# ANY MANUAL ACTION.
+#
+# That said, you are welcome to correct your name or add a nickname / GitHub
+# user name as appropriate. The format is:
+#
+#    Name Name Name (nickname) <email1@example.com> <email2@example.com>
+#
+# The in-GUI authors list is periodically automatically updated from the
+# contents of this file.
+#
 
-Aaron Bieber <qbit@deftly.net>
-Adam Piggott <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com>
-Alessandro G. <alessandro.g89@gmail.com>
-Alexander Graf <register-github@alex-graf.de>
-Anderson Mesquita <andersonvom@gmail.com>
-Andrew Dunham <andrew@du.nham.ca>
-Antony Male <antony.male@gmail.com>
-Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
-Alexandre Viau <alexandre@alexandreviau.net> <aviau@debian.org>
-Audrius Butkevicius <audrius.butkevicius@gmail.com>
-Bart De Vries <devriesb@gmail.com>
-Ben Curthoys <ben@bencurthoys.com>
-Ben Schulz <ueomkail@gmail.com> <uok@users.noreply.github.com>
-Ben Sidhom <bsidhom@gmail.com>
-Benny Ng <benny.tpng@gmail.com>
-Brandon Philips <brandon@ifup.org>
-Brendan Long <self@brendanlong.com>
-Brian R. Becker <brbecker@gmail.com>
-Caleb Callaway <enlightened.despot@gmail.com>
-Carsten Hagemann <moter8@gmail.com>
-Cathryne Linenweaver <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
-Chris Howie <me@chrishowie.com>
-Chris Joel <chris@scriptolo.gy>
-Colin Kennedy <moshen.colin@gmail.com>
-Daniel Bergmann <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
-Daniel Harte <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
-Daniel Martí <mvdan@mvdan.cc>
-David Rimmer <dinosore@dbrsoftware.co.uk>
-Denis A. <denisva@gmail.com>
-Dennis Wilson <dw@risu.io>
-Dominik Heidler <dominik@heidler.eu>
-Elias Jarlebring <jarlebring@gmail.com>
-Emil Hessman <emil@hessman.se>
-Erik Meitner <e.meitner@willystreet.coop>
-Federico Castagnini <federico.castagnini@gmail.com>
-Felix Ableitner <me@nutomic.com>
-Felix Unterpaintner <bigbear2nd@gmail.com>
-Francois-Xavier Gsell <fxgsell@gmail.com>
-Frank Isemann <frank@isemann.name>
-Gilli Sigurdsson <gilli@vx.is>
-Jaakko Hannikainen <jgke@jgke.fi>
-Jacek Szafarkiewicz <szafar@linux.pl>
-Jake Peterson <jake@acogdev.com>
-Jakob Borg <jakob@nym.se>
-James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
-Jaroslav Malec <dzardacz@gmail.com>
-Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
-Jochen Voss <voss@seehuhn.de>
-Johan Vromans <jvromans@squirrel.nl>
-Karol Różycki <rozycki.karol@gmail.com>
-Kelong Cong <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
-Ken'ichi Kamada <kamada@nanohz.org>
-Kevin Allen <kma1660@gmail.com>
-Lars K.W. Gohlke <lkwg82@gmx.de>
-Laurent Etiemble <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
-Lode Hoste <zillode@zillode.be>
-Lord Landon Agahnim <lordlandon@gmail.com>
-Marc Laporte <marc@marclaporte.com> <marc@laporte.name>
-Marc Pujol <kilburn@la3.org>
-Marcin Dziadus <dziadus.marcin@gmail.com>
-Mateusz Naściszewski <matin1111@wp.pl>
-Matt Burke <mburke@amplify.com> <burkemw3@gmail.com>
-Max Schulze <max.schulze@online.de> <kralo@users.noreply.github.com>
-Michael Jephcote <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
-Michael Ploujnikov <ploujj@gmail.com>
-Michael Tilli <pyfisch@gmail.com>
-Nate Morrison <natemorrison@gmail.com>
-Pascal Jungblut <github@pascalj.com> <mail@pascal-jungblut.com>
-Peter Hoeg <peter@speartail.com>
-Philippe Schommers <philippe@schommers.be>
-Phill Luby <phill.luby@newredo.com>
-Piotr Bejda <piotrb10@gmail.com>
-Ryan Sullivan <kayoticsully@gmail.com>
-Scott Klupfel <kluppy@going2blue.com>
-Sergey Mishin <ralder@yandex.ru>
-Stefan Kuntz <stefan.github@gmail.com> <Stefan.github@gmail.com>
-Stefan Tatschner <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
-Tim Abell <tim@timwise.co.uk>
-Tobias Nygren <tnn@nygren.pp.se>
-Tomas Cerveny <kozec@kozec.com>
-Tully Robinson <tully@tojr.org>
-Tyler Brazier <tyler@tylerbrazier.com>
-Veeti Paananen <veeti.paananen@rojekti.fi>
-Victor Buinsky <vix_booja@tut.by>
-Vil Brekin <vilbrekin@gmail.com>
-William A. Kennington III <william@wkennington.com>
-Wulf Weich <wweich@users.noreply.github.com> <wweich@gmx.de>
-Yannic A. <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
+Aaron Bieber (qbit) <qbit@deftly.net>
+Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com> <adam@proactiveservices.co.uk>
+Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
+Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
+Alexander Graf (alex2108) <register-github@alex-graf.de>
+Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
+Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
+andresvia <andres.via@gmail.com>
+Andrew Dunham (andrew-d) <andrew@du.nham.ca>
+Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
+Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
+André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
+andyleap <andyleap@gmail.com>
+Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
+Antony Male (canton7) <antony.male@gmail.com>
+Aranjedeath <Aranjedeath@users.noreply.github.com>
+Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
+Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
+BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
+Bart De Vries (mogwa1) <devriesb@gmail.com>
+Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
+Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
+Ben Shepherd (benshep) <bjashepherd@gmail.com>
+Ben Sidhom (bsidhom) <bsidhom@gmail.com>
+Benedikt Heine (bebehei) <bebe@bebehei.de>
+Benedikt Morbach <benedikt.morbach@googlemail.com>
+Benno Fünfstück <benno.fuenfstueck@gmail.com>
+Benny Ng (tpng) <benny.tpng@gmail.com>
+Boris Rybalkin <ribalkin@gmail.com>
+Brandon Philips (philips) <brandon@ifup.org>
+Brendan Long (brendanlong) <self@brendanlong.com>
+Brian R. Becker (brbecker) <brbecker@gmail.com>
+Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
+Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
+Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
+Cedric Staniewski (xduugu) <cedric@gmx.ca>
+Chris Howie (cdhowie) <me@chrishowie.com>
+Chris Joel (cdata) <chris@scriptolo.gy>
+Chris Tonkinson <chris@masterbran.ch>
+chucic <chucic@seznam.cz>
+Colin Kennedy (moshen) <moshen.colin@gmail.com>
+Cromefire_ <tim.l@nghorst.net>
+Dale Visser <dale.visser@live.com>
+Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
+Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
+Daniel Martí (mvdan) <mvdan@mvdan.cc>
+Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
+David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
+Denis A. (dva) <denisva@gmail.com>
+Dennis Wilson (snnd) <dw@risu.io>
+dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com>
+dependabot[bot] <dependabot[bot]@users.noreply.github.com>
+derekriemer <derek.riemer@colorado.edu>
+desbma <desbma@users.noreply.github.com>
+Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
+Dominik Heidler (asdil12) <dominik@heidler.eu>
+Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
+Elliot Huffman <thelich2@gmail.com>
+Emil Hessman (ceh) <emil@hessman.se>
+Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
+Evgeny Kuznetsov <evgeny@kuznetsov.md>
+Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
+Felix Ableitner (Nutomic) <me@nutomic.com>
+Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
+Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
+Frank Isemann (fti7) <frank@isemann.name>
+georgespatton <georgespatton@users.noreply.github.com>
+Gilli Sigurdsson (gillisig) <gilli@vx.is>
+Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
+Han Boetes <han@boetes.org>
+Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
+Heiko Zuerker (Smiley73) <heiko@zuerker.org>
+Hugo Locurcio <hugo.locurcio@hugo.pro>
+Iain Barnett <iainspeed@gmail.com>
+Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
+Iskander Sharipov (Alex) <quasilyte@gmail.com>
+Jaakko Hannikainen (jgke) <jgke@jgke.fi>
+Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
+Jake Peterson (acogdev) <jake@acogdev.com>
+Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
+James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
+janost <janost@tuta.io>
+Jaroslav Malec (dzarda) <dzardacz@gmail.com>
+jaseg <githubaccount@jaseg.net>
+Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
+Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
+Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
+Jochen Voss (seehuhn) <voss@seehuhn.de>
+Johan Andersson <j@i19.se>
+Johan Vromans (sciurius) <jvromans@squirrel.nl>
+John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
+Jonas Thelemann <e-mail@jonas-thelemann.de>
+Jonathan Cross <jcross@gmail.com>
+Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
+Jörg Thalheim <Mic92@users.noreply.github.com>
+Kalle Laine <pahakalle@protonmail.com>
+Karol Różycki (krozycki) <rozycki.karol@gmail.com>
+Keith Turner <kturner@apache.org>
+Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
+Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
+Kevin Allen (ironmig) <kma1660@gmail.com>
+Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
+klemens <ka7@github.com>
+Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
+Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
+Laurent Arnoud <laurent@spkdev.net>
+Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
+Leo Arias (elopio) <yo@elopio.net>
+Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
+Lode Hoste (Zillode) <zillode@zillode.be>
+Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
+Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
+Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
+Marc Pujol (kilburn) <kilburn@la3.org>
+Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
+marco-m <marco.molteni@laposte.net>
+Mark Pulford (mpx) <mark@kyne.com.au>
+Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
+Matic Potočnik <hairyfotr@gmail.com>
+Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
+Matt Robenolt <matt@ydekproductions.com>
+Matteo Ruina <matteo.ruina@gmail.com>
+Maurizio Tomasi <ziotom78@gmail.com>
+Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
+MaximAL <almaximal@ya.ru>
+Maxime Thirouin <m@moox.io>
+Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
+Michael Ploujnikov (plouj) <ploujj@gmail.com>
+Michael Tilli (pyfisch) <pyfisch@gmail.com>
+Mike Boone <mike@boonedocks.net>
+MikeLund <MikeLund@users.noreply.github.com>
+Mingxuan Lin <gdlmx@users.noreply.github.com>
+Nate Morrison (nrm21) <natemorrison@gmail.com>
+Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
+Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
+Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
+Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
+Nils Jakobi (thunderstorm99) <jakobi.nils@gmail.com>
+Nitroretro <43112364+Nitroretro@users.noreply.github.com>
+NoLooseEnds <jon.koslung@gmail.com>
+otbutz <tbutz@optitool.de>
+Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
+Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
+Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
+Paweł Rozlach <vespian@users.noreply.github.com>
+perewa <cavalcante.ten@gmail.com>
+Peter Badida <KeyWeeUsr@users.noreply.github.com>
+Peter Dave Hello <hsu@peterdavehello.org>
+Peter Hoeg (peterhoeg) <peter@speartail.com>
+Peter Marquardt (wwwutz) <wwwutz@gmail.com> <wwwutz@googlemail.com>
+Phil Davis <phil.davis@inf.org>
+Philippe Schommers (filoozoom) <philippe@schommers.be>
+Phill Luby (pluby) <phill.luby@newredo.com>
+Pier Paolo Ramon <ramonpierre@gmail.com>
+Piotr Bejda (piobpl) <piotrb10@gmail.com>
+Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
+Richard Hartmann <RichiH@users.noreply.github.com>
+Robert Carosi (nov1n) <robert@carosi.nl>
+Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
+Ross Smith II (rasa) <ross@smithii.com>
+rubenbe <github-com-00ff86@vandamme.email>
+Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
+Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
+Scott Klupfel (kluppy) <kluppy@going2blue.com>
+Sergey Mishin (ralder) <ralder@yandex.ru>
+Simon Frei (imsodin) <freisim93@gmail.com>
+Sly_tom_cat <slytomcat@mail.ru>
+Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
+Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
+Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
+Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
+Thomas Hipp <thomashipp@gmail.com>
+Tim Abell (timabell) <tim@timwise.co.uk>
+Tim Howes (timhowes) <timhowes@berkeley.edu>
+Tobias Nygren (tnn2) <tnn@nygren.pp.se>
+Tobias Tom (tobiastom) <t.tom@succont.de>
+Tom Jakubowski <tom@crystae.net>
+Tomas Cerveny (kozec) <kozec@kozec.com>
+Tommy Thorn <tommy-github-email@thorn.ws>
+Tully Robinson (tojrobinson) <tully@tojr.org>
+Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
+Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
+Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
+Victor Buinsky (buinsky) <vix_booja@tut.by>
+Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
+Vladimir Rusinov <vrusinov@google.com>
+wangguoliang <liangcszzu@163.com>
+William A. Kennington III (wkennington) <william@wkennington.com>
+Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
+Xavier O. (damajor) <damajor@gmail.com>
+xjtdy888 (xjtdy888) <xjtdy888@163.com>
+Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
+佛跳墙 <daoquan@qq.com>

CONDUCT.md

@@ -1,92 +1,73 @@
-## Conduct
+# Contributor Covenant Code of Conduct
 
-* We are committed to providing a friendly, safe and welcoming
-  environment for all, regardless of gender, sexual orientation,
-  disability, ethnicity, religion, or similar personal characteristic.
+## Our Pledge
 
-* On IRC, please avoid using overtly sexual nicknames or other nicknames
-  that might detract from a friendly, safe and welcoming environment for
-  all.
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+education, socio-economic status, nationality, personal appearance, race,
+religion, or sexual identity and orientation.
 
-* Please be kind and courteous. There's no need to be mean or rude.
+## Our Standards
 
-* Respect that people have differences of opinion and that every design
-  or implementation choice carries a trade-off and numerous costs. There
-  is seldom a right answer.
+Examples of behavior that contributes to creating a positive environment
+include:
 
-* Please keep unstructured critique to a minimum. If you have solid
-  ideas you want to experiment with, make a fork and see how it works.
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
 
-* We will exclude you from interaction if you insult, demean or harass
-  anyone. That is not welcome behaviour. We interpret the term
-  "harassment" as including the definition in the <a
-  href="http://citizencodeofconduct.org/">Citizen Code of Conduct</a>;
-  if you have any lack of clarity about what might be included in that
-  concept, please read their definition. In particular, we don't
-  tolerate behavior that excludes people in socially marginalized
-  groups.
+Examples of unacceptable behavior by participants include:
 
-* Private harassment is also unacceptable. No matter who you are, if you
-  feel you have been or are being harassed or made uncomfortable by a
-  community member, please contact one of the channel ops or any of the
-  Syncthing core team immediately. Whether you're a regular  contributor
-  or a newcomer, we care about making this community a safe place for
-  you and we've got your back.
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
 
-* Likewise any spamming, trolling, flaming, baiting or other
-  attention-stealing behaviour is not welcome.
+## Our Responsibilities
 
-## Moderation
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
 
-These are the policies for upholding our community's standards of
-conduct in our communication channels, most notably in Syncthing-related
-IRC channels and on the web forum.
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
 
-1. Remarks that violate the Syncthing standards of conduct, including
-   hateful, hurtful, oppressive, or exclusionary remarks, are not
-   allowed. (Cursing is allowed, but never targeting another user, and
-   never in a hateful manner.)
+## Scope
 
-2. Remarks that moderators find inappropriate, whether listed in the
-   code of conduct or not, are also not allowed.
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
 
-3. Moderators will first respond to such remarks with a warning.
+## Enforcement
 
-4. If the warning is unheeded, the user will be "kicked," i.e., kicked
-   out of the communication channel to cool off.
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at security@syncthing.net. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
 
-5. If the user comes back and continues to make trouble, they will be
-   banned, i.e., indefinitely excluded.
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
 
-6. Moderators may choose at their discretion to un-ban the user if it
-   was a first offense and they offer the offended party a genuine
-   apology.
+## Attribution
 
-7. If a moderator bans someone and you think it was unjustified, please
-   take it up with that moderator, or with a different moderator, **in
-   private**. Complaints about bans in-channel are not allowed.
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
 
-8. Moderators are held to a higher standard than other community
-   members. If a moderator creates an inappropriate situation, they
-   should expect less leeway than others.
-
-In the Syncthing community we strive to go the extra step to look out
-for each other. Don't just aim to be technically unimpeachable, try to
-be your best self. In particular, avoid flirting with offensive or
-sensitive issues, particularly if they're off-topic; this all too
-often leads to unnecessary fights, hurt feelings, and damaged trust;
-worse, it can drive people away from the community entirely.
-
-And if someone takes issue with something you said or did, resist the
-urge to be defensive. Just stop doing what it was they complained about
-and apologize. Even if you feel you were misinterpreted or unfairly
-accused, chances are good there was something you could've communicated
-better — remember that it's your responsibility to make your fellow
-community members comfortable. Everyone wants to get along and we are
-all here first and foremost because we want to talk about cool
-technology. You will find that people will be eager to assume good
-intent and forgive as long as you earn their trust.
-
-*Adapted from the [Rust Code of Conduct](https://github.com/rust-lang/rust/wiki/Note-development-policy#conduct)*
-
-*Adapted from the [Node.js Policy on Trolling](http://blog.izs.me/post/30036893703/policy-on-trolling)*
+[homepage]: https://www.contributor-covenant.org

CONTRIBUTING.md

@@ -33,20 +33,31 @@ latest info on Transifex.
 
 Every contribution is welcome. If you want to contribute but are unsure
 where to start, any open issues are fair game! See the [Contribution
-Guidelines](http://docs.syncthing.net/dev/contributing.html) for the full
+Guidelines](https://docs.syncthing.net/dev/contributing.html) for the full
 story on committing code.
 
 ## Contributing Documentation
 
-Updates to the [documentation site](http://docs.syncthing.net/) can be
+Updates to the [documentation site](https://docs.syncthing.net/) can be
 made as pull requests on the [documentation
 repository](https://github.com/syncthing/docs).
 
 ## Licensing
 
-All contributions are made under the same MPLv2 license as the rest of
-the project, except documentation, user interface text and translation
-strings which are licensed under the Creative Commons Attribution 4.0
-International License. You retain the copyright to code you have
-written.
+All contributions are made available under the same license as the already
+existing material being contributed to. For most of the project and unless
+otherwise stated this means MPLv2, but there are exceptions:
+
+- Certain commands (under cmd/...) may have a separate license, indicated by
+  the presence of a LICENSE file in the corresponding directory.
+
+- The documentation (man/...) is licensed under the Creative Commons
+  Attribution 4.0 International License.
+
+- Projects under vendor/... are copyright by and licensed from their
+  respective original authors. Contributions should be made to the original
+  project, not here.
+
+Regardless of the license in effect, you retain the copyright to your
+contribution.
 

Dockerfile

@@ -0,0 +1,28 @@
+FROM golang:1.13 AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f syncthing && go run build.go -no-upgrade build syncthing
+
+FROM alpine
+
+EXPOSE 8384 22000 21027/udp
+
+VOLUME ["/var/syncthing"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/syncthing /bin/syncthing
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 8384 || exit 1
+
+ENV STGUIADDRESS=0.0.0.0:8384
+ENTRYPOINT ["/bin/entrypoint.sh", "-home", "/var/syncthing/config"]

GOALS.md

@@ -0,0 +1,83 @@
+# The Syncthing Goals
+
+Syncthing is a **continuous file synchronization program**. It synchronizes
+files between two or more computers. We strive to fulfill the goals below.
+The goals are listed in order of importance, the most important one being
+the first.
+
+> "Syncing files" here is precise. It means we specifically exclude things
+> that are not files - calendar items, instant messages, and so on. If those
+> are in fact stored as files on disk, they can of course be synced as
+> files.
+
+Syncthing should be:
+
+### 1. Safe From Data Loss
+
+Protecting the user's data is paramount. We take every reasonable precaution
+to avoid corrupting the user's files.
+
+> This is the overriding goal, without which synchronizing files becomes
+> pointless. This means that we do not make unsafe trade offs for the sake
+> of performance or, in some cases, even usability.
+
+### 2. Secure Against Attackers
+
+Again, protecting the user's data is paramount. Regardless of our other
+goals we must never allow the user's data to be susceptible to eavesdropping
+or modification by unauthorized parties.
+
+> This should be understood in context. It is not necessarily reasonable to
+> expect Syncthing to be resistant against well equipped state level
+> attackers. We will however do our best. Note also that this is different
+> from anonymity which is not, currently, a goal.
+
+### 3. Easy to Use
+
+Syncthing should be approachable, understandable and inclusive.
+
+> Complex concepts and maths form the base of Syncthing's functionality.
+> This should nonetheless be abstracted or hidden to a degree where
+> Syncthing is usable by the general public.
+
+### 4. Automatic
+
+User interaction should be required only when absolutely necessary.
+
+> Specifically this means that changes to files are picked up without
+> prompting, conflicts are resolved without prompting and connections are
+> maintained without prompting. We only prompt the user when it is required
+> to fulfill one of the (overriding) Secure, Safe or Easy goals.
+
+### 5. Universally Available
+
+Syncthing should run on every common computer. We are mindful that the
+latest technology is not always available to any given individual.
+
+> Computers include desktops, laptops, servers, virtual machines, small
+> general purpose computers such as Raspberry Pis and, *where possible*,
+> tablets and phones. NAS appliances, toasters, cars, firearms, thermostats
+> and so on may include computing capabitilies but it is not our goal for
+> Syncthing to run smoothly on these devices.
+
+### 6. For Individuals
+
+Syncthing is primarily about empowering the individual user with safe,
+secure and easy to use file synchronization.
+
+> We acknowledge that it's also useful in an enterprise setting and include
+> functionality to support that. If this is in conflict with the
+> requirements of the individual, those will however take priority.
+
+### 7. Everything Else
+
+There are many things we care about that don't make it on to the list. It is
+fine to optimize for these values as well, as long as they are not in
+conflict with the stated goals above.
+
+> For example, performance is a thing we care about. We just don't care more
+> about it than safety, security, etc. Maintainability of the code base and
+> providing entertainment value for the maintainers are also things that
+> matter. It is understood that there are aspects of Syncthing that are
+> suboptimal or even in opposition with the goals above. However, we
+> continuously strive to align Syncthing more and more with these goals.

ISSUE_TEMPLATE.md

@@ -1,28 +0,0 @@
-Do not report security issues in this bug tracker. Instead, contact
-security@syncthing.net directly - see https://syncthing.net/security.html
-for more information.
-
-If your issue is a support request ("How do I get my devices to connect?"
-or similar), please use the support forum at https://forum.syncthing.net/
-where a large number of helpful people hang out. This issue tracker is for
-reporting bugs or feature requests directly to the developers.
-
-If your issue is a bug report, replace this boilerplate with a description
-of the problem, being sure to include at least:
-
- - what happened,
- - what you expected to happen instead, and
- - any steps to reproduce the problem.
-
-Also fill out the version information below and add log output or
-screenshots as appropriate.
-
-If your issue is a feature request, simply replace this template text in
-its entirety.
-
-### Version Information
-
-Syncthing Version: v0.x.y
-OS Version: Windows 7 / Ubuntu 14.04 / ...
-Browser Version: (if applicable, for GUI issues)
-

LICENSE

@@ -357,7 +357,7 @@ Exhibit A - Source Code Form License Notice
 
   This Source Code Form is subject to the terms of the Mozilla Public
   License, v. 2.0. If a copy of the MPL was not distributed with this
-  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+  file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 If it is not possible or desirable to put the notice in a particular
 file, then You may include the notice in a location (such as a LICENSE

NICKS

@@ -1,89 +0,0 @@
-# This file maps email addresses used in commits to nicks used the changelog.
-
-acogdev		<jake@acogdev.com>
-alex2108	<register-github@alex-graf.de>
-alessandro.g89	<alessandro.g89@gmail.com>
-andersonvom	<andersonvom@gmail.com>
-andrew-d	<andrew@du.nham.ca>
-asdil12		<dominik@heidler.eu>
-AudriusButkevicius	<audrius.butkevicius@gmail.com>
-aviau       <alexandre@alexandreviau.net> <aviau@debian.org>
-bencurthoys	<ben@bencurthoys.com>
-bigbear2nd	<bigbear2nd@gmail.com>
-brbecker	<brbecker@gmail.com>
-brendanlong	<self@brendanlong.com>
-brgmnn      <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
-bsidhom		<bsidhom@gmail.com>
-buinsky	<vix_booja@tut.by>
-burkemw3	<mburke@amplify.com> <burkemw3@gmail.com>
-calmh		<jakob@nym.se>
-canton7	<antony.male@gmail.com>
-Cathryne	<cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
-cdata		<chris@scriptolo.gy>
-cdhowie	<me@chrishowie.com>
-ceh		<emil@hessman.se>
-cqcallaw	<enlightened.despot@gmail.com>
-dinosore	<dinosore@dbrsoftware.co.uk>
-dva	<denisva@gmail.com>
-dzarda		<dzardacz@gmail.com>
-eipiminus1	<eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
-facastagnini	<federico.castagnini@gmail.com>
-filoozoom	<philippe@schommers.be>
-frioux		<frew@afoolishmanifesto.com> <frioux@gmail.com>
-fti7	<frank@isemann.name>
-gillisig	<gilli@vx.is>
-hadogenes	<szafar@linux.pl>
-ironmig	<kma1660@gmail.com>
-jarlebring	<jarlebring@gmail.com>
-jedie		<github.com@jensdiemer.de> <git@jensdiemer.de>
-jgke	<jgke@jgke.fi>
-jpjp		<jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
-kamadak		<kamada@nanohz.org>
-KayoticSully	<kayoticsully@gmail.com>
-kilburn		<kilburn@la3.org>
-kluppy	<kluppy@going2blue.com>
-kozec		<kozec@kozec.com>
-kralo	<max.schulze@online.de>
-krozycki	<rozycki.karol@gmail.com>
-letiemble	<laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
-LordLandon	<lordlandon@gmail.com>
-lkwg82	<lkwg82@gmx.de>
-marcindziadus	<dziadus.marcin@gmail.com>
-marclaporte <marc@marclaporte.com>
-mateon1	<matin1111@wp.pl>
-mogwa1	<devriesb@gmail.com>
-moshen		<moshen.colin@gmail.com>
-Moter8		<moter8@gmail.com>
-mvdan		<mvdan@mvdan.cc>
-norgeous	<daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
-nrm21	<natemorrison@gmail.com>
-Nutomic		<me@nutomic.com>
-pascalj		<github@pascalj.com> <mail@pascal-jungblut.com>
-peterhoeg	<peter@speartail.com>
-philips		<brandon@ifup.org>
-piobpl		<piotrb10@gmail.com>
-plouj	<ploujj@gmail.com>
-pluby		<phill.luby@newredo.com>
-pyfisch		<pyfisch@gmail.com>
-qbit		<qbit@deftly.net>
-ralder		<ralder@yandex.ru>
-Rewt0r		<rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
-rumpelsepp	<stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
-sciurius	<jvromans@squirrel.nl>
-seehuhn		<voss@seehuhn.de>
-simplypeachy	<aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com>
-snnd		<dw@risu.io>
-Stefan-Code	<stefan.github@gmail.com> <Stefan.github@gmail.com>
-timabell	<tim@timwise.co.uk>
-tnn2		<tnn@nygren.pp.se>
-tojrobinson	<tully@tojr.org>
-tpng		<benny.tpng@gmail.com>
-tylerbrazier	<tyler@tylerbrazier.com>
-uok		<ueomkail@gmail.com> <uok@users.noreply.github.com>
-veeti		<veeti.paananen@rojekti.fi>
-Vilbrekin	<vilbrekin@gmail.com>
-wkennington	<william@wkennington.com>
-wsgcsysadmin	<e.meitner@willystreet.coo>
-wweich	<wweich@users.noreply.github.com> <wweich@gmx.de>
-Zillode		<zillode@zillode.be>
-zukoo		<fxgsell@gmail.com>

README-Docker.md

@@ -0,0 +1,59 @@
+# Docker Container for Syncthing
+
+Use the Dockerfile in this repo, or pull the `syncthing/syncthing` image
+from Docker Hub.
+
+Use the `/var/syncthing` volume to have the synchronized files available on the
+host. You can add more folders and map them as you prefer.
+
+Note that Syncthing runs as UID 1000 and GID 1000 by default. These may be
+altered with the ``PUID`` and ``PGID`` environment variables.
+
+## Example Usage
+
+```
+$ docker pull syncthing/syncthing
+$ docker run -p 8384:8384 -p 22000:22000 \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+## Discovery
+
+Note that local device discovery will not work with the above command,
+resulting in poor local transfer rates if local device addresses are not
+manually configured.
+
+To allow local discovery, the docker host network can be used instead:
+
+```
+$ docker pull syncthing/syncthing
+$ docker run --network=host \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+Be aware that syncthing alone is now in control of what interfaces and ports it
+listens on. You can edit the syncthing configuration to change the defaults if
+there are conflicts.
+
+## GUI Security
+
+By default Syncthing inside the Docker image listens on 0.0.0.0:8384 to
+allow GUI connections via the Docker proxy. This is set by the
+`STGUIADDRESS` environment variable in the Dockerfile, as it differs from
+what Syncthing would otherwise use by default. This means you should set up
+authentication in the GUI, like for any other externally reachable Syncthing
+instance. If you do not require the GUI, or you use host networking, you can
+unset the `STGUIADDRESS` variable to have Syncthing fall back to listening
+on 127.0.0.1:
+
+```
+$ docker pull syncthing/syncthing
+$ docker run -e STGUIADDRESS= \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+With the environment variable unset Syncthing will follow what is set in the
+configuration file / GUI settings dialog.

README.md

@@ -1,23 +1,58 @@
-# Syncthing
+[![Syncthing][14]][15]
 
-[![Latest Build (Official)](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square&label=unix%20build)](http://build.syncthing.net/job/syncthing/lastBuild/)
-[![API Documentation](https://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/syncthing)
+---
+
+[![Latest Linux & Cross Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildLinuxCross.svg?style=flat-square&label=linux+%26+cross+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildLinuxCross&guest=1)
+[![Latest Windows Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildWindows.svg?style=flat-square&label=windows+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildWindows&guest=1)
+[![Latest Mac Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildMac.svg?style=flat-square&label=mac+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildMac&guest=1)
 [![MPLv2 License](https://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/88/badge)](https://bestpractices.coreinfrastructure.org/projects/88)
+[![Go Report Card](https://goreportcard.com/badge/github.com/syncthing/syncthing)](https://goreportcard.com/report/github.com/syncthing/syncthing)
 
-This is the Syncthing project which pursues the following goals:
+## Goals
 
- 1. Define a protocol for synchronization of a folder between a number of
-    collaborating devices. This protocol should be well defined, unambiguous,
-    easily understood, free to use, efficient, secure and language neutral.
-    This is called the [Block Exchange Protocol][1].
+Syncthing is a **continuous file synchronization program**. It synchronizes
+files between two or more computers. We strive to fulfill the goals below.
+The goals are listed in order of importance, the most important one being
+the first. This is the summary version of the goal list - for more
+commentary, see the full [Goals document][13].
 
- 2. Provide the reference implementation to demonstrate the usability of
-    said protocol. This is the `syncthing` utility. We hope that
-    alternative, compatible implementations of the protocol will arise.
+Syncthing should be:
 
-The two are evolving together; the protocol is not to be considered
-stable until Syncthing 1.0 is released, at which point it is locked down
-for incompatible changes.
+1. **Safe From Data Loss**
+
+   Protecting the user's data is paramount. We take every reasonable
+   precaution to avoid corrupting the user's files.
+
+2. **Secure Against Attackers**
+
+   Again, protecting the user's data is paramount. Regardless of our other
+   goals we must never allow the user's data to be susceptible to
+   eavesdropping or modification by unauthorized parties.
+
+3. **Easy to Use**
+
+   Syncthing should be approachable, understandable and inclusive.
+
+4. **Automatic**
+
+   User interaction should be required only when absolutely necessary.
+
+5. **Universally Available**
+
+   Syncthing should run on every common computer. We are mindful that the
+   latest technology is not always available to any given individual.
+
+6. **For Individuals**
+
+   Syncthing is primarily about empowering the individual user with safe,
+   secure and easy to use file synchronization.
+
+7. **Everything Else**
+
+   There are many things we care about that don't make it on to the list. It
+   is fine to optimize for these values, as long as they are not in conflict
+   with the stated goals above.
 
 ## Getting Started
 
@@ -27,6 +62,15 @@ There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory][3]. There are also several [GUI
 implementations][11] for Windows, Mac and Linux.
 
+## Docker
+
+To run Syncthing in Docker, see [the Docker README][16].
+
+## Vote on features/bugs
+
+We'd like to encourage you to [vote][12] on issues that matter to you.
+This helps the team understand what are the biggest pain points for our users, and could potentially influence what is being worked on next.
+
 ## Getting in Touch
 
 The first and best point of contact is the [Forum][8]. There is also an IRC
@@ -46,8 +90,8 @@ D26E6ED000654A3E, available from https://syncthing.net/security.html and
 most key servers.
 
 There is also a built in automatic upgrade mechanism (disabled in some
-distribution channels) which uses a compiled in ECDSA signature. Mac OS
-X binaries are also properly code signed.
+distribution channels) which uses a compiled in ECDSA signature. macOS
+binaries are also properly code signed.
 
 ## Documentation
 
@@ -55,14 +99,20 @@ Please see the [Syncthing documentation site][6].
 
 All code is licensed under the [MPLv2 License][7].
 
-[1]: http://docs.syncthing.net/specs/bep-v1.html
-[2]: http://docs.syncthing.net/intro/getting-started.html
+[1]: https://docs.syncthing.net/specs/bep-v1.html
+[2]: https://docs.syncthing.net/intro/getting-started.html
 [3]: https://github.com/syncthing/syncthing/blob/master/etc
-[4]: http://www.freenode.net/irc_servers.shtml
-[5]: http://docs.syncthing.net/dev/building.html
-[6]: http://docs.syncthing.net/
+[4]: https://www.freenode.net/
+[5]: https://docs.syncthing.net/dev/building.html
+[6]: https://docs.syncthing.net/
 [7]: https://github.com/syncthing/syncthing/blob/master/LICENSE
 [8]: https://forum.syncthing.net/
 [9]: https://kiwiirc.com/client/irc.freenode.net/#syncthing
 [10]: https://github.com/syncthing/syncthing/issues
-[11]: http://docs.syncthing.net/users/contrib.html#gui-wrappers
+[11]: https://docs.syncthing.net/users/contrib.html#gui-wrappers
+[12]: https://www.bountysource.com/teams/syncthing/issues
+[13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
+[14]: assets/logo-text-128.png
+[15]: https://syncthing.net/
+[16]: https://github.com/syncthing/syncthing/blob/master/README-Docker.md
+

build.sh

@@ -48,9 +48,6 @@ case "${1:-default}" in
 		;;
 
 	test)
-		ulimit -t 600 &>/dev/null || true
-		ulimit -d 512000 &>/dev/null || true
-		ulimit -m 512000 &>/dev/null || true
 		LOGGER_DISCARD=1 build test
 		;;
 
@@ -61,9 +58,9 @@ case "${1:-default}" in
 	prerelease)
 		go run script/authors.go
 		build transifex
-		git add -A gui/default/assets/ lib/auto/
 		pushd man ; ./refresh.sh ; popd
-		git add -A man
+		git add -A gui man AUTHORS
+		git commit -m 'gui, man, authors: Update docs, translations, and contributors'
 		;;
 
 	noupgrade)
@@ -93,93 +90,12 @@ case "${1:-default}" in
 		done
 		;;
 
-	test-cov)
-		ulimit -t 600 &>/dev/null || true
-		ulimit -d 512000 &>/dev/null || true
-		ulimit -m 512000 &>/dev/null || true
-
-		echo "mode: set" > coverage.out
-		fail=0
-
-		# For every package in the repo
-		for dir in $(go list ./lib/... ./cmd/...) ; do
-			# run the tests
-			GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -race -coverprofile=profile.out $dir
-			if [ -f profile.out ] ; then
-				# and if there was test output, append it to coverage.out
-				grep -v "mode: " profile.out >> coverage.out
-				rm profile.out
-			fi
-		done
-
-		gocov convert coverage.out | gocov-xml > coverage.xml
-
-		# This is usually run from within Jenkins. If it is, we need to
-		# tweak the paths in coverage.xml so cobertura finds the
-		# source.
-		if [[ "${WORKSPACE:-default}" != "default" ]] ; then
-			sed "s#$WORKSPACE##g" < coverage.xml > coverage.xml.new && mv coverage.xml.new coverage.xml
-		fi
-		;;
-
 	test-xunit)
-		ulimit -t 600 &>/dev/null || true
-		ulimit -d 512000 &>/dev/null || true
-		ulimit -m 512000 &>/dev/null || true
 
 		(GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -v -race ./lib/... ./cmd/... || true) > tests.out
 		go2xunit -output tests.xml -fail < tests.out
 		;;
 
-	docker-all)
-		img=${DOCKERIMG:-syncthing/build:latest}
-		docker run --rm -h syncthing-builder -u $(id -u) -t \
-			-v $(pwd):/go/src/github.com/syncthing/syncthing \
-			-w /go/src/github.com/syncthing/syncthing \
-			-e "STTRACE=$STTRACE" \
-			"$img" \
-			sh -c './build.sh clean \
-				&& ./build.sh test-cov \
-				&& ./build.sh bench \
-				&& ./build.sh all'
-		;;
-
-	docker-test)
-		img=${DOCKERIMG:-syncthing/build:latest}
-		docker run --rm -h syncthing-builder -u $(id -u) -t \
-			-v $(pwd):/go/src/github.com/syncthing/syncthing \
-			-w /go/src/github.com/syncthing/syncthing \
-			-e "STTRACE=$STTRACE" \
-			"$img" \
-			sh -euxc './build.sh clean \
-				&& go run build.go -race \
-				&& export GOPATH=$(pwd)/Godeps/_workspace:$GOPATH \
-				&& cd test \
-				&& go test -tags integration -v -timeout 90m -short \
-				&& git clean -fxd .'
-		;;
-
-	docker-lint)
-		img=${DOCKERIMG:-syncthing/build:latest}
-		docker run --rm -h syncthing-builder -u $(id -u) -t \
-			-v $(pwd):/go/src/github.com/syncthing/syncthing \
-			-w /go/src/github.com/syncthing/syncthing \
-			-e "STTRACE=$STTRACE" \
-			"$img" \
-			sh -euxc 'go run build.go lint'
-		;;
-
-
-	docker-vet)
-		img=${DOCKERIMG:-syncthing/build:latest}
-		docker run --rm -h syncthing-builder -u $(id -u) -t \
-			-v $(pwd):/go/src/github.com/syncthing/syncthing \
-			-w /go/src/github.com/syncthing/syncthing \
-			-e "STTRACE=$STTRACE" \
-			"$img" \
-			sh -euxc 'go run build.go vet'
-		;;
-
 	*)
 		echo "Unknown build command $1"
 		;;

cmd/stbench/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 // This doesn't build on Windows due to the Rusage stuff.
 
@@ -92,7 +92,7 @@ loop:
 	return t1.Sub(t0)
 }
 
-// report stops the given process and reports on it's resource usage in two
+// report stops the given process and reports on its resource usage in two
 // ways: human readable to stderr, and CSV to stdout.
 func report(p *rc.Process, wallTime time.Duration) {
 	sv, err := p.SystemVersion()

cmd/stcli/client.go

@@ -0,0 +1,95 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+
+	"github.com/syncthing/syncthing/lib/config"
+)
+
+type APIClient struct {
+	http.Client
+	cfg    config.GUIConfiguration
+	apikey string
+}
+
+func getClient(cfg config.GUIConfiguration) *APIClient {
+	httpClient := http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial(cfg.Network(), cfg.Address())
+			},
+		},
+	}
+	return &APIClient{
+		Client: httpClient,
+		cfg:    cfg,
+		apikey: cfg.APIKey,
+	}
+}
+
+func (c *APIClient) Endpoint() string {
+	if c.cfg.Network() == "unix" {
+		return "http://unix/"
+	}
+	url := c.cfg.URL()
+	if !strings.HasSuffix(url, "/") {
+		url += "/"
+	}
+	return url
+}
+
+func (c *APIClient) Do(req *http.Request) (*http.Response, error) {
+	req.Header.Set("X-API-Key", c.apikey)
+	resp, err := c.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	return resp, checkResponse(resp)
+}
+
+func (c *APIClient) Get(url string) (*http.Response, error) {
+	request, err := http.NewRequest("GET", c.Endpoint()+"rest/"+url, nil)
+	if err != nil {
+		return nil, err
+	}
+	return c.Do(request)
+}
+
+func (c *APIClient) Post(url, body string) (*http.Response, error) {
+	request, err := http.NewRequest("POST", c.Endpoint()+"rest/"+url, bytes.NewBufferString(body))
+	if err != nil {
+		return nil, err
+	}
+	return c.Do(request)
+}
+
+func checkResponse(response *http.Response) error {
+	if response.StatusCode == 404 {
+		return fmt.Errorf("Invalid endpoint or API call")
+	} else if response.StatusCode == 403 {
+		return fmt.Errorf("Invalid API key")
+	} else if response.StatusCode != 200 {
+		data, err := responseToBArray(response)
+		if err != nil {
+			return err
+		}
+		body := strings.TrimSpace(string(data))
+		return fmt.Errorf("Unexpected HTTP status returned: %s\n%s", response.Status, body)
+	}
+	return nil
+}

cmd/stcli/errors.go

@@ -0,0 +1,60 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/urfave/cli"
+)
+
+var errorsCommand = cli.Command{
+	Name:     "errors",
+	HideHelp: true,
+	Usage:    "Error command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "show",
+			Usage:  "Show pending errors",
+			Action: expects(0, dumpOutput("system/error")),
+		},
+		{
+			Name:      "push",
+			Usage:     "Push an error to active clients",
+			ArgsUsage: "[error message]",
+			Action:    expects(1, errorsPush),
+		},
+		{
+			Name:   "clear",
+			Usage:  "Clear pending errors",
+			Action: expects(0, emptyPost("system/error/clear")),
+		},
+	},
+}
+
+func errorsPush(c *cli.Context) error {
+	client := c.App.Metadata["client"].(*APIClient)
+	errStr := strings.Join(c.Args(), " ")
+	response, err := client.Post("system/error", strings.TrimSpace(errStr))
+	if err != nil {
+		return err
+	}
+	if response.StatusCode != 200 {
+		errStr = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
+		bytes, err := responseToBArray(response)
+		if err != nil {
+			return err
+		}
+		body := string(bytes)
+		if body != "" {
+			errStr += "\nBody: " + body
+		}
+		return fmt.Errorf(errStr)
+	}
+	return nil
+}

cmd/stcli/main.go

@@ -0,0 +1,192 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bufio"
+	"crypto/tls"
+	"encoding/json"
+	"flag"
+	"log"
+	"os"
+	"reflect"
+
+	"github.com/AudriusButkevicius/recli"
+	"github.com/flynn-archive/go-shlex"
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/locations"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/urfave/cli"
+)
+
+func main() {
+	// This is somewhat a hack around a chicken and egg problem.
+	// We need to set the home directory and potentially other flags to know where the syncthing instance is running
+	// in order to get it's config ... which we then use to construct the actual CLI ... at which point it's too late
+	// to add flags there...
+	homeBaseDir := locations.GetBaseDir(locations.ConfigBaseDir)
+	guiCfg := config.GUIConfiguration{}
+
+	flags := flag.NewFlagSet("", flag.ContinueOnError)
+	flags.StringVar(&guiCfg.RawAddress, "gui-address", guiCfg.RawAddress, "Override GUI address (e.g. \"http://192.0.2.42:8443\")")
+	flags.StringVar(&guiCfg.APIKey, "gui-apikey", guiCfg.APIKey, "Override GUI API key")
+	flags.StringVar(&homeBaseDir, "home", homeBaseDir, "Set configuration directory")
+
+	// Implement the same flags at the lower CLI, with the same default values (pre-parse), but do nothing with them.
+	// This is so that we could reuse os.Args
+	fakeFlags := []cli.Flag{
+		cli.StringFlag{
+			Name:  "gui-address",
+			Value: guiCfg.RawAddress,
+			Usage: "Override GUI address (e.g. \"http://192.0.2.42:8443\")",
+		},
+		cli.StringFlag{
+			Name:  "gui-apikey",
+			Value: guiCfg.APIKey,
+			Usage: "Override GUI API key",
+		},
+		cli.StringFlag{
+			Name:  "home",
+			Value: homeBaseDir,
+			Usage: "Set configuration directory",
+		},
+	}
+
+	// Do not print usage of these flags, and ignore errors as this can't understand plenty of things
+	flags.Usage = func() {}
+	_ = flags.Parse(os.Args[1:])
+
+	// Now if the API key and address is not provided (we are not connecting to a remote instance),
+	// try to rip it out of the config.
+	if guiCfg.RawAddress == "" && guiCfg.APIKey == "" {
+		// Update the base directory
+		err := locations.SetBaseDir(locations.ConfigBaseDir, homeBaseDir)
+		if err != nil {
+			log.Fatal(errors.Wrap(err, "setting home"))
+		}
+
+		// Load the certs and get the ID
+		cert, err := tls.LoadX509KeyPair(
+			locations.Get(locations.CertFile),
+			locations.Get(locations.KeyFile),
+		)
+		if err != nil {
+			log.Fatal(errors.Wrap(err, "reading device ID"))
+		}
+
+		myID := protocol.NewDeviceID(cert.Certificate[0])
+
+		// Load the config
+		cfg, err := config.Load(locations.Get(locations.ConfigFile), myID, events.NoopLogger)
+		if err != nil {
+			log.Fatalln(errors.Wrap(err, "loading config"))
+		}
+
+		guiCfg = cfg.GUI()
+	} else if guiCfg.Address() == "" || guiCfg.APIKey == "" {
+		log.Fatalln("Both -gui-address and -gui-apikey should be specified")
+	}
+
+	if guiCfg.Address() == "" {
+		log.Fatalln("Could not find GUI Address")
+	}
+
+	if guiCfg.APIKey == "" {
+		log.Fatalln("Could not find GUI API key")
+	}
+
+	client := getClient(guiCfg)
+
+	cfg, err := getConfig(client)
+	original := cfg.Copy()
+	if err != nil {
+		log.Fatalln(errors.Wrap(err, "getting config"))
+	}
+
+	// Copy the config and set the default flags
+	recliCfg := recli.DefaultConfig
+	recliCfg.IDTag.Name = "xml"
+	recliCfg.SkipTag.Name = "json"
+
+	commands, err := recli.New(recliCfg).Construct(&cfg)
+	if err != nil {
+		log.Fatalln(errors.Wrap(err, "config reflect"))
+	}
+
+	// Construct the actual CLI
+	app := cli.NewApp()
+	app.Name = "stcli"
+	app.HelpName = app.Name
+	app.Author = "The Syncthing Authors"
+	app.Usage = "Syncthing command line interface"
+	app.Version = build.Version
+	app.Flags = fakeFlags
+	app.Metadata = map[string]interface{}{
+		"client": client,
+	}
+	app.Commands = []cli.Command{
+		{
+			Name:        "config",
+			HideHelp:    true,
+			Usage:       "Configuration modification command group",
+			Subcommands: commands,
+		},
+		showCommand,
+		operationCommand,
+		errorsCommand,
+	}
+
+	tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
+	if !tty {
+		// Not a TTY, consume from stdin
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			input, err := shlex.Split(scanner.Text())
+			if err != nil {
+				log.Fatalln(errors.Wrap(err, "parsing input"))
+			}
+			if len(input) == 0 {
+				continue
+			}
+			err = app.Run(append(os.Args, input...))
+			if err != nil {
+				log.Fatalln(err)
+			}
+		}
+		err = scanner.Err()
+		if err != nil {
+			log.Fatalln(err)
+		}
+	} else {
+		err = app.Run(os.Args)
+		if err != nil {
+			log.Fatalln(err)
+		}
+	}
+
+	if !reflect.DeepEqual(cfg, original) {
+		body, err := json.MarshalIndent(cfg, "", "  ")
+		if err != nil {
+			log.Fatalln(err)
+		}
+		resp, err := client.Post("system/config", string(body))
+		if err != nil {
+			log.Fatalln(err)
+		}
+		if resp.StatusCode != 200 {
+			body, err := responseToBArray(resp)
+			if err != nil {
+				log.Fatalln(err)
+			}
+			log.Fatalln(string(body))
+		}
+	}
+}

cmd/stcli/operations.go

@@ -0,0 +1,78 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli"
+)
+
+var operationCommand = cli.Command{
+	Name:     "operations",
+	HideHelp: true,
+	Usage:    "Operation command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "restart",
+			Usage:  "Restart syncthing",
+			Action: expects(0, emptyPost("system/restart")),
+		},
+		{
+			Name:   "shutdown",
+			Usage:  "Shutdown syncthing",
+			Action: expects(0, emptyPost("system/shutdown")),
+		},
+		{
+			Name:   "reset",
+			Usage:  "Reset syncthing deleting all folders and devices",
+			Action: expects(0, emptyPost("system/reset")),
+		},
+		{
+			Name:   "upgrade",
+			Usage:  "Upgrade syncthing (if a newer version is available)",
+			Action: expects(0, emptyPost("system/upgrade")),
+		},
+		{
+			Name:      "folder-override",
+			Usage:     "Override changes on folder (remote for sendonly, local for receiveonly)",
+			ArgsUsage: "[folder id]",
+			Action:    expects(1, foldersOverride),
+		},
+	},
+}
+
+func foldersOverride(c *cli.Context) error {
+	client := c.App.Metadata["client"].(*APIClient)
+	cfg, err := getConfig(client)
+	if err != nil {
+		return err
+	}
+	rid := c.Args()[0]
+	for _, folder := range cfg.Folders {
+		if folder.ID == rid {
+			response, err := client.Post("db/override", "")
+			if err != nil {
+				return err
+			}
+			if response.StatusCode != 200 {
+				errStr := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
+				bytes, err := responseToBArray(response)
+				if err != nil {
+					return err
+				}
+				body := string(bytes)
+				if body != "" {
+					errStr += "\nBody: " + body
+				}
+				return fmt.Errorf(errStr)
+			}
+			return nil
+		}
+	}
+	return fmt.Errorf("Folder " + rid + " not found")
+}

cmd/stcli/show.go

@@ -0,0 +1,44 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"github.com/urfave/cli"
+)
+
+var showCommand = cli.Command{
+	Name:     "show",
+	HideHelp: true,
+	Usage:    "Show command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "version",
+			Usage:  "Show syncthing client version",
+			Action: expects(0, dumpOutput("system/version")),
+		},
+		{
+			Name:   "config-status",
+			Usage:  "Show configuration status, whether or not a restart is required for changes to take effect",
+			Action: expects(0, dumpOutput("system/config/insync")),
+		},
+		{
+			Name:   "system",
+			Usage:  "Show system status",
+			Action: expects(0, dumpOutput("system/status")),
+		},
+		{
+			Name:   "connections",
+			Usage:  "Report about connections to other devices",
+			Action: expects(0, dumpOutput("system/connections")),
+		},
+		{
+			Name:   "usage",
+			Usage:  "Show usage report",
+			Action: expects(0, dumpOutput("svc/report")),
+		},
+	},
+}

cmd/stcli/utils.go

@@ -0,0 +1,101 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"text/tabwriter"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/urfave/cli"
+)
+
+func responseToBArray(response *http.Response) ([]byte, error) {
+	bytes, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return nil, err
+	}
+	return bytes, response.Body.Close()
+}
+
+func emptyPost(url string) cli.ActionFunc {
+	return func(c *cli.Context) error {
+		client := c.App.Metadata["client"].(*APIClient)
+		_, err := client.Post(url, "")
+		return err
+	}
+}
+
+func dumpOutput(url string) cli.ActionFunc {
+	return func(c *cli.Context) error {
+		client := c.App.Metadata["client"].(*APIClient)
+		response, err := client.Get(url)
+		if err != nil {
+			return err
+		}
+		return prettyPrintResponse(c, response)
+	}
+}
+
+func newTableWriter() *tabwriter.Writer {
+	writer := new(tabwriter.Writer)
+	writer.Init(os.Stdout, 0, 8, 0, '\t', 0)
+	return writer
+}
+
+func getConfig(c *APIClient) (config.Configuration, error) {
+	cfg := config.Configuration{}
+	response, err := c.Get("system/config")
+	if err != nil {
+		return cfg, err
+	}
+	bytes, err := responseToBArray(response)
+	if err != nil {
+		return cfg, err
+	}
+	err = json.Unmarshal(bytes, &cfg)
+	if err == nil {
+		return cfg, err
+	}
+	return cfg, nil
+}
+
+func expects(n int, actionFunc cli.ActionFunc) cli.ActionFunc {
+	return func(ctx *cli.Context) error {
+		if ctx.NArg() != n {
+			plural := ""
+			if n != 1 {
+				plural = "s"
+			}
+			return fmt.Errorf("expected %d argument%s, got %d", n, plural, ctx.NArg())
+		}
+		return actionFunc(ctx)
+	}
+}
+
+func prettyPrintJSON(data interface{}) error {
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	return enc.Encode(data)
+}
+
+func prettyPrintResponse(c *cli.Context, response *http.Response) error {
+	bytes, err := responseToBArray(response)
+	if err != nil {
+		return err
+	}
+	var data interface{}
+	if err := json.Unmarshal(bytes, &data); err != nil {
+		return err
+	}
+	// TODO: Check flag for pretty print format
+	return prettyPrintJSON(data)
+}

cmd/stcompdirs/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -15,8 +15,6 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-
-	"github.com/syncthing/syncthing/lib/symlinks"
 )
 
 func main() {
@@ -90,10 +88,10 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 		}
 
 		rn, _ := filepath.Rel(dir, path)
-		if rn == "." || rn == ".stfolder" {
+		if rn == "." {
 			return nil
 		}
-		if rn == ".stversions" {
+		if rn == ".stversions" || rn == ".stfolder" {
 			return filepath.SkipDir
 		}
 
@@ -104,7 +102,7 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 				mode: os.ModeSymlink,
 			}
 
-			tgt, _, err := symlinks.Read(path)
+			tgt, err := os.Readlink(path)
 			if err != nil {
 				return err
 			}

cmd/stcrashreceiver/sentry.go

@@ -0,0 +1,204 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"regexp"
+	"strings"
+	"sync"
+
+	raven "github.com/getsentry/raven-go"
+	"github.com/maruel/panicparse/stack"
+)
+
+const reportServer = "https://crash.syncthing.net/report/"
+
+var loader = newGithubSourceCodeLoader()
+
+func init() {
+	raven.SetSourceCodeLoader(loader)
+}
+
+var (
+	clients    = make(map[string]*raven.Client)
+	clientsMut sync.Mutex
+)
+
+func sendReport(dsn, path string, report []byte) error {
+	pkt, err := parseReport(path, report)
+	if err != nil {
+		return err
+	}
+
+	clientsMut.Lock()
+	defer clientsMut.Unlock()
+
+	cli, ok := clients[dsn]
+	if !ok {
+		cli, err = raven.New(dsn)
+		if err != nil {
+			return err
+		}
+		clients[dsn] = cli
+	}
+
+	// The client sets release and such on the packet before sending, in the
+	// misguided idea that it knows this better than than the packet we give
+	// it. So we copy the values from the packet to the client first...
+	cli.SetRelease(pkt.Release)
+	cli.SetEnvironment(pkt.Environment)
+
+	defer cli.Wait()
+	_, errC := cli.Capture(pkt, nil)
+	return <-errC
+}
+
+func parseReport(path string, report []byte) (*raven.Packet, error) {
+	parts := bytes.SplitN(report, []byte("\n"), 2)
+	if len(parts) != 2 {
+		return nil, errors.New("no first line")
+	}
+
+	version, err := parseVersion(string(parts[0]))
+	if err != nil {
+		return nil, err
+	}
+	report = parts[1]
+
+	foundPanic := false
+	var subjectLine []byte
+	for {
+		parts = bytes.SplitN(report, []byte("\n"), 2)
+		if len(parts) != 2 {
+			return nil, errors.New("no panic line found")
+		}
+
+		line := parts[0]
+		report = parts[1]
+
+		if foundPanic {
+			// The previous line was our "Panic at ..." header. We are now
+			// at the beginning of the real panic trace and this is our
+			// subject line.
+			subjectLine = line
+			break
+		} else if bytes.HasPrefix(line, []byte("Panic at")) {
+			foundPanic = true
+		}
+	}
+
+	r := bytes.NewReader(report)
+	ctx, err := stack.ParseDump(r, ioutil.Discard, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// Lock the source code loader to the version we are processing here.
+	if version.commit != "" {
+		// We have a commit hash, so we know exactly which source to use
+		loader.LockWithVersion(version.commit)
+	} else if strings.HasPrefix(version.tag, "v") {
+		// Lets hope the tag is close enough
+		loader.LockWithVersion(version.tag)
+	} else {
+		// Last resort
+		loader.LockWithVersion("master")
+	}
+	defer loader.Unlock()
+
+	var trace raven.Stacktrace
+	for _, gr := range ctx.Goroutines {
+		if gr.First {
+			trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
+			for i, sc := range gr.Stack.Calls {
+				trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name(), sc.SrcPath, sc.Line, 3, nil)
+			}
+			break
+		}
+	}
+
+	pkt := &raven.Packet{
+		Message:     string(subjectLine),
+		Platform:    "go",
+		Release:     version.tag,
+		Environment: version.environment(),
+		Tags: raven.Tags{
+			raven.Tag{Key: "version", Value: version.version},
+			raven.Tag{Key: "tag", Value: version.tag},
+			raven.Tag{Key: "codename", Value: version.codename},
+			raven.Tag{Key: "runtime", Value: version.runtime},
+			raven.Tag{Key: "goos", Value: version.goos},
+			raven.Tag{Key: "goarch", Value: version.goarch},
+			raven.Tag{Key: "builder", Value: version.builder},
+		},
+		Extra: raven.Extra{
+			"url": reportServer + path,
+		},
+		Interfaces: []raven.Interface{&trace},
+	}
+	if version.commit != "" {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
+	}
+
+	return pkt, nil
+}
+
+// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC
+var longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)`)
+
+type version struct {
+	version  string // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
+	tag      string // "v1.1.4-rc.1"
+	commit   string // "6aaae618", blank when absent
+	codename string // "Erbium Earthworm"
+	runtime  string // "go1.12.5"
+	goos     string // "darwin"
+	goarch   string // "amd64"
+	builder  string // "jb@kvin.kastelo.net"
+}
+
+func (v version) environment() string {
+	if v.commit != "" {
+		return "Development"
+	}
+	if strings.Contains(v.tag, "-rc.") {
+		return "Candidate"
+	}
+	if strings.Contains(v.tag, "-") {
+		return "Beta"
+	}
+	return "Stable"
+}
+
+func parseVersion(line string) (version, error) {
+	m := longVersionRE.FindStringSubmatch(line)
+	if len(m) == 0 {
+		return version{}, errors.New("unintelligeble version string")
+	}
+
+	v := version{
+		version:  m[1],
+		codename: m[2],
+		runtime:  m[3],
+		goos:     m[4],
+		goarch:   m[5],
+		builder:  m[6],
+	}
+	parts := strings.Split(v.version, "+")
+	v.tag = parts[0]
+	if len(parts) > 1 {
+		fields := strings.Split(parts[1], "-")
+		if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
+			v.commit = fields[1][1:]
+		}
+	}
+
+	return v, nil
+}

cmd/stcrashreceiver/sentry_test.go

@@ -0,0 +1,64 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+)
+
+func TestParseVersion(t *testing.T) {
+	cases := []struct {
+		longVersion string
+		parsed      version
+	}{
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		v, err := parseVersion(tc.longVersion)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		if v != tc.parsed {
+			t.Error(v)
+		}
+	}
+}
+
+func TestParseReport(t *testing.T) {
+	bs, err := ioutil.ReadFile("_testdata/panic.log")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pkt, err := parseReport("1/2/345", bs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bs, err = pkt.JSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Printf("%s\n", bs)
+}

cmd/stcrashreceiver/sourcecodeloader.go

@@ -0,0 +1,114 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	urlPrefix   = "https://raw.githubusercontent.com/syncthing/syncthing/"
+	httpTimeout = 10 * time.Second
+)
+
+type githubSourceCodeLoader struct {
+	mut     sync.Mutex
+	version string
+	cache   map[string]map[string][][]byte // version -> file -> lines
+	client  *http.Client
+}
+
+func newGithubSourceCodeLoader() *githubSourceCodeLoader {
+	return &githubSourceCodeLoader{
+		cache:  make(map[string]map[string][][]byte),
+		client: &http.Client{Timeout: httpTimeout},
+	}
+}
+
+func (l *githubSourceCodeLoader) LockWithVersion(version string) {
+	l.mut.Lock()
+	l.version = version
+	if _, ok := l.cache[version]; !ok {
+		l.cache[version] = make(map[string][][]byte)
+	}
+}
+
+func (l *githubSourceCodeLoader) Unlock() {
+	l.mut.Unlock()
+}
+
+func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]byte, int) {
+	filename = filepath.ToSlash(filename)
+	lines, ok := l.cache[l.version][filename]
+	if !ok {
+		// Cache whatever we managed to find (or nil if nothing, so we don't try again)
+		defer func() {
+			l.cache[l.version][filename] = lines
+		}()
+
+		knownPrefixes := []string{"/lib/", "/cmd/"}
+		var idx int
+		for _, pref := range knownPrefixes {
+			idx = strings.Index(filename, pref)
+			if idx >= 0 {
+				break
+			}
+		}
+		if idx == -1 {
+			return nil, 0
+		}
+
+		url := urlPrefix + l.version + filename[idx:]
+		resp, err := l.client.Get(url)
+
+		if err != nil || resp.StatusCode != http.StatusOK {
+			fmt.Println("Loading source:", err.Error())
+			return nil, 0
+		}
+		data, err := ioutil.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		if err != nil {
+			fmt.Println("Loading source:", err.Error())
+			return nil, 0
+		}
+		lines = bytes.Split(data, []byte{'\n'})
+	}
+
+	return getLineFromLines(lines, line, context)
+}
+
+func getLineFromLines(lines [][]byte, line, context int) ([][]byte, int) {
+	if lines == nil {
+		// cached error from ReadFile: return no lines
+		return nil, 0
+	}
+
+	line-- // stack trace lines are 1-indexed
+	start := line - context
+	var idx int
+	if start < 0 {
+		start = 0
+		idx = line
+	} else {
+		idx = context
+	}
+	end := line + context + 1
+	if line >= len(lines) {
+		return nil, 0
+	}
+	if end > len(lines) {
+		end = len(lines)
+	}
+	return lines[start:end], idx
+}

cmd/stcrashreceiver/stcrashreceiver.go

@@ -0,0 +1,160 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Command stcrashreceiver is a trivial HTTP server that allows two things:
+//
+// - uploading files (crash reports) named like a SHA256 hash using a PUT request
+// - checking whether such file exists using a HEAD request
+//
+// Typically this should be deployed behind something that manages HTTPS.
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"flag"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+const maxRequestSize = 1 << 20 // 1 MiB
+
+func main() {
+	dir := flag.String("dir", ".", "Directory to store reports in")
+	dsn := flag.String("dsn", "", "Sentry DSN")
+	listen := flag.String("listen", ":22039", "HTTP listen address")
+	flag.Parse()
+
+	cr := &crashReceiver{
+		dir: *dir,
+		dsn: *dsn,
+	}
+
+	log.SetOutput(os.Stdout)
+	if err := http.ListenAndServe(*listen, cr); err != nil {
+		log.Fatalln("HTTP serve:", err)
+	}
+}
+
+type crashReceiver struct {
+	dir string
+	dsn string
+}
+
+func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	// The final path component should be a SHA256 hash in hex, so 64 hex
+	// characters. We don't care about case on the request but use lower
+	// case internally.
+	reportID := strings.ToLower(path.Base(req.URL.Path))
+	if len(reportID) != 64 {
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+	for _, c := range reportID {
+		if c >= 'a' && c <= 'f' {
+			continue
+		}
+		if c >= '0' && c <= '9' {
+			continue
+		}
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+
+	// The location of the report on disk, compressed
+	fullPath := filepath.Join(r.dir, r.dirFor(reportID), reportID) + ".gz"
+
+	switch req.Method {
+	case http.MethodGet:
+		r.serveGet(fullPath, w, req)
+	case http.MethodHead:
+		r.serveHead(fullPath, w, req)
+	case http.MethodPut:
+		r.servePut(reportID, fullPath, w, req)
+	default:
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+// serveGet responds to GET requests by serving the uncompressed report.
+func (r *crashReceiver) serveGet(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	fd, err := os.Open(fullPath)
+	if err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+		return
+	}
+
+	defer fd.Close()
+	gr, err := gzip.NewReader(fd)
+	if err != nil {
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+	_, _ = io.Copy(w, gr) // best effort
+}
+
+// serveHead responds to HEAD requests by checking if the named report
+// already exists in the system.
+func (r *crashReceiver) serveHead(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	if _, err := os.Lstat(fullPath); err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+	}
+}
+
+// servePut accepts and stores the given report.
+func (r *crashReceiver) servePut(reportID, fullPath string, w http.ResponseWriter, req *http.Request) {
+	// Ensure the destination directory exists
+	if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
+		log.Println("Creating directory:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Read at most maxRequestSize of report data.
+	log.Println("Receiving report", reportID)
+	lr := io.LimitReader(req.Body, maxRequestSize)
+	bs, err := ioutil.ReadAll(lr)
+	if err != nil {
+		log.Println("Reading report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Compress the report for storage
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	_, _ = gw.Write(bs) // can't fail
+	gw.Close()
+
+	// Create an output file with the compressed report
+	err = ioutil.WriteFile(fullPath, buf.Bytes(), 0644)
+	if err != nil {
+		log.Println("Saving report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Send the report to Sentry
+	if r.dsn != "" {
+		go func() {
+			// There's no need for the client to have to wait for this part.
+			if err := sendReport(r.dsn, reportID, bs); err != nil {
+				log.Println("Failed to send report:", err)
+			}
+		}()
+	}
+}
+
+// 01234567890abcdef... => 01/23
+func (r *crashReceiver) dirFor(base string) string {
+	return filepath.Join(base[0:2], base[2:4])
+}

cmd/stdisco/main.go

@@ -2,13 +2,13 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
-	"bytes"
 	"crypto/rand"
+	"encoding/binary"
 	"flag"
 	"log"
 	"strings"
@@ -44,7 +44,7 @@ func main() {
 	flag.Parse()
 
 	if fake {
-		log.Println("My ID:", protocol.DeviceIDFromBytes(myID))
+		log.Println("My ID:", myID)
 	}
 
 	runbeacon(beacon.NewMulticast(mc), fake)
@@ -66,24 +66,25 @@ func recv(bc beacon.Interface) {
 	seen := make(map[string]bool)
 	for {
 		data, src := bc.Recv()
-		var ann discover.Announce
-		ann.UnmarshalXDR(data)
+		if m := binary.BigEndian.Uint32(data); m != discover.Magic {
+			log.Printf("Incorrect magic %x in announcement from %v", m, src)
+			continue
+		}
 
-		if bytes.Equal(ann.This.ID, myID) {
+		var ann discover.Announce
+		ann.Unmarshal(data[4:])
+
+		if ann.ID == myID {
 			// This is one of our own fake packets, don't print it.
 			continue
 		}
 
 		// Print announcement details for the first packet from a given
 		// device ID and source address, or if -all was given.
-		key := string(ann.This.ID) + src.String()
+		key := ann.ID.String() + src.String()
 		if all || !seen[key] {
 			log.Printf("Announcement from %v\n", src)
-			log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(ann.This.ID), strings.Join(addrStrs(ann.This), ", "))
-
-			for _, dev := range ann.Extra {
-				log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(dev.ID), strings.Join(addrStrs(dev), ", "))
-			}
+			log.Printf(" %v at %s\n", ann.ID, strings.Join(ann.Addresses, ", "))
 			seen[key] = true
 		}
 	}
@@ -92,15 +93,10 @@ func recv(bc beacon.Interface) {
 // sends fake discovery announcements once every second
 func send(bc beacon.Interface) {
 	ann := discover.Announce{
-		Magic: discover.AnnouncementMagic,
-		This: discover.Device{
-			ID: myID,
-			Addresses: []discover.Address{
-				{URL: "tcp://fake.example.com:12345"},
-			},
-		},
+		ID:        myID,
+		Addresses: []string{"tcp://fake.example.com:12345"},
 	}
-	bs, _ := ann.MarshalXDR()
+	bs, _ := ann.Marshal()
 
 	for {
 		bc.Send(bs)
@@ -108,19 +104,10 @@ func send(bc beacon.Interface) {
 	}
 }
 
-// returns the list of address URLs
-func addrStrs(dev discover.Device) []string {
-	ss := make([]string, len(dev.Addresses))
-	for i, addr := range dev.Addresses {
-		ss[i] = addr.URL
-	}
-	return ss
-}
-
 // returns a random but recognizable device ID
-func randomDeviceID() []byte {
-	var id [32]byte
+func randomDeviceID() protocol.DeviceID {
+	var id protocol.DeviceID
 	copy(id[:], randomPrefix)
 	rand.Read(id[len(randomPrefix):])
-	return id[:]
+	return id
 }

cmd/stdiscosrv/README.md

@@ -0,0 +1,10 @@
+stdiscosrv
+==========
+
+This is the global discovery server for the `syncthing` project.
+
+Usage
+-----
+
+https://docs.syncthing.net/users/stdiscosrv.html
+

cmd/stdiscosrv/apisrv.go

@@ -0,0 +1,411 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"log"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+// announcement is the format received from and sent to clients
+type announcement struct {
+	Seen      time.Time `json:"seen"`
+	Addresses []string  `json:"addresses"`
+}
+
+type apiSrv struct {
+	addr     string
+	cert     tls.Certificate
+	db       database
+	listener net.Listener
+	repl     replicator // optional
+	useHTTP  bool
+
+	mapsMut sync.Mutex
+	misses  map[string]int32
+}
+
+type requestID int64
+
+func (i requestID) String() string {
+	return fmt.Sprintf("%016x", int64(i))
+}
+
+type contextKey int
+
+const idKey contextKey = iota
+
+func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP bool) *apiSrv {
+	return &apiSrv{
+		addr:    addr,
+		cert:    cert,
+		db:      db,
+		repl:    repl,
+		useHTTP: useHTTP,
+		misses:  make(map[string]int32),
+	}
+}
+
+func (s *apiSrv) Serve() {
+	if s.useHTTP {
+		listener, err := net.Listen("tcp", s.addr)
+		if err != nil {
+			log.Println("Listen:", err)
+			return
+		}
+		s.listener = listener
+	} else {
+		tlsCfg := &tls.Config{
+			Certificates:           []tls.Certificate{s.cert},
+			ClientAuth:             tls.RequestClientCert,
+			SessionTicketsDisabled: true,
+			MinVersion:             tls.VersionTLS12,
+			CipherSuites: []uint16{
+				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+			},
+		}
+
+		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
+		if err != nil {
+			log.Println("Listen:", err)
+			return
+		}
+		s.listener = tlsListener
+	}
+
+	http.HandleFunc("/", s.handler)
+	http.HandleFunc("/ping", handlePing)
+
+	srv := &http.Server{
+		ReadTimeout:    httpReadTimeout,
+		WriteTimeout:   httpWriteTimeout,
+		MaxHeaderBytes: httpMaxHeaderBytes,
+	}
+
+	if err := srv.Serve(s.listener); err != nil {
+		log.Println("Serve:", err)
+	}
+}
+
+var topCtx = context.Background()
+
+func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
+	t0 := time.Now()
+
+	lw := NewLoggingResponseWriter(w)
+
+	defer func() {
+		diff := time.Since(t0)
+		apiRequestsSeconds.WithLabelValues(req.Method).Observe(diff.Seconds())
+		apiRequestsTotal.WithLabelValues(req.Method, strconv.Itoa(lw.statusCode)).Inc()
+	}()
+
+	reqID := requestID(rand.Int63())
+	ctx := context.WithValue(topCtx, idKey, reqID)
+
+	if debug {
+		log.Println(reqID, req.Method, req.URL)
+	}
+
+	var remoteIP net.IP
+	if s.useHTTP {
+		remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
+	} else {
+		addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
+		if err != nil {
+			log.Println("remoteAddr:", err)
+			lw.Header().Set("Retry-After", errorRetryAfterString())
+			http.Error(lw, "Internal Server Error", http.StatusInternalServerError)
+			apiRequestsTotal.WithLabelValues("no_remote_addr").Inc()
+			return
+		}
+		remoteIP = addr.IP
+	}
+
+	switch req.Method {
+	case "GET":
+		s.handleGET(ctx, lw, req)
+	case "POST":
+		s.handlePOST(ctx, remoteIP, lw, req)
+	default:
+		http.Error(lw, "Method Not Allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *apiSrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
+	reqID := ctx.Value(idKey).(requestID)
+
+	deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
+	if err != nil {
+		if debug {
+			log.Println(reqID, "bad device param")
+		}
+		lookupRequestsTotal.WithLabelValues("bad_request").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	key := deviceID.String()
+	rec, err := s.db.get(key)
+	if err != nil {
+		// some sort of internal error
+		lookupRequestsTotal.WithLabelValues("internal_error").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+
+	if len(rec.Addresses) == 0 {
+		lookupRequestsTotal.WithLabelValues("not_found").Inc()
+
+		s.mapsMut.Lock()
+		misses := s.misses[key]
+		if misses < rec.Misses {
+			misses = rec.Misses + 1
+		} else {
+			misses++
+		}
+		s.misses[key] = misses
+		s.mapsMut.Unlock()
+
+		if misses%notFoundMissesWriteInterval == 0 {
+			rec.Misses = misses
+			rec.Missed = time.Now().UnixNano()
+			rec.Addresses = nil
+			// rec.Seen retained from get
+			s.db.put(key, rec)
+		}
+
+		w.Header().Set("Retry-After", notFoundRetryAfterString(int(misses)))
+		http.Error(w, "Not Found", http.StatusNotFound)
+		return
+	}
+
+	lookupRequestsTotal.WithLabelValues("success").Inc()
+
+	bs, _ := json.Marshal(announcement{
+		Seen:      time.Unix(0, rec.Seen),
+		Addresses: addressStrs(rec.Addresses),
+	})
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bs)
+}
+
+func (s *apiSrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
+	reqID := ctx.Value(idKey).(requestID)
+
+	rawCert := certificateBytes(req)
+	if rawCert == nil {
+		if debug {
+			log.Println(reqID, "no certificates")
+		}
+		announceRequestsTotal.WithLabelValues("no_certificate").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Forbidden", http.StatusForbidden)
+		return
+	}
+
+	var ann announcement
+	if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
+		if debug {
+			log.Println(reqID, "decode:", err)
+		}
+		announceRequestsTotal.WithLabelValues("bad_request").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	deviceID := protocol.NewDeviceID(rawCert)
+
+	addresses := fixupAddresses(remoteIP, ann.Addresses)
+	if len(addresses) == 0 {
+		announceRequestsTotal.WithLabelValues("bad_request").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	if err := s.handleAnnounce(remoteIP, deviceID, addresses); err != nil {
+		announceRequestsTotal.WithLabelValues("internal_error").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+
+	announceRequestsTotal.WithLabelValues("success").Inc()
+
+	w.Header().Set("Reannounce-After", reannounceAfterString())
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func (s *apiSrv) Stop() {
+	s.listener.Close()
+}
+
+func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addresses []string) error {
+	key := deviceID.String()
+	now := time.Now()
+	expire := now.Add(addressExpiryTime).UnixNano()
+
+	dbAddrs := make([]DatabaseAddress, len(addresses))
+	for i := range addresses {
+		dbAddrs[i].Address = addresses[i]
+		dbAddrs[i].Expires = expire
+	}
+
+	seen := now.UnixNano()
+	if s.repl != nil {
+		s.repl.send(key, dbAddrs, seen)
+	}
+	return s.db.merge(key, dbAddrs, seen)
+}
+
+func handlePing(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(204)
+}
+
+func certificateBytes(req *http.Request) []byte {
+	if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
+		return req.TLS.PeerCertificates[0].Raw
+	}
+
+	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
+		bs := []byte(hdr)
+		// The certificate is in PEM format but with spaces for newlines. We
+		// need to reinstate the newlines for the PEM decoder. But we need to
+		// leave the spaces in the BEGIN and END lines - the first and last
+		// space - alone.
+		firstSpace := bytes.Index(bs, []byte(" "))
+		lastSpace := bytes.LastIndex(bs, []byte(" "))
+		for i := firstSpace + 1; i < lastSpace; i++ {
+			if bs[i] == ' ' {
+				bs[i] = '\n'
+			}
+		}
+		block, _ := pem.Decode(bs)
+		if block == nil {
+			// Decoding failed
+			return nil
+		}
+		return block.Bytes
+	}
+
+	return nil
+}
+
+// fixupAddresses checks the list of addresses, removing invalid ones and
+// replacing unspecified IPs with the given remote IP.
+func fixupAddresses(remote net.IP, addresses []string) []string {
+	fixed := make([]string, 0, len(addresses))
+	for _, annAddr := range addresses {
+		uri, err := url.Parse(annAddr)
+		if err != nil {
+			continue
+		}
+
+		host, port, err := net.SplitHostPort(uri.Host)
+		if err != nil {
+			continue
+		}
+
+		ip := net.ParseIP(host)
+
+		// Some classes of IP are no-go.
+		if ip.IsLoopback() || ip.IsMulticast() {
+			continue
+		}
+
+		if host == "" || ip.IsUnspecified() {
+			// Replace the unspecified IP with the request source.
+
+			// ... unless the request source is the loopback address or
+			// multicast/unspecified (can't happen, really).
+			if remote.IsLoopback() || remote.IsMulticast() || remote.IsUnspecified() {
+				continue
+			}
+
+			// Do not use IPv6 remote address if requested scheme is ...4
+			// (i.e., tcp4, etc.)
+			if strings.HasSuffix(uri.Scheme, "4") && remote.To4() == nil {
+				continue
+			}
+
+			// Do not use IPv4 remote address if requested scheme is ...6
+			if strings.HasSuffix(uri.Scheme, "6") && remote.To4() != nil {
+				continue
+			}
+
+			host = remote.String()
+		}
+
+		uri.Host = net.JoinHostPort(host, port)
+		fixed = append(fixed, uri.String())
+	}
+
+	return fixed
+}
+
+type loggingResponseWriter struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
+	return &loggingResponseWriter{w, http.StatusOK}
+}
+
+func (lrw *loggingResponseWriter) WriteHeader(code int) {
+	lrw.statusCode = code
+	lrw.ResponseWriter.WriteHeader(code)
+}
+
+func addressStrs(dbAddrs []DatabaseAddress) []string {
+	res := make([]string, len(dbAddrs))
+	for i, a := range dbAddrs {
+		res[i] = a.Address
+	}
+	return res
+}
+
+func errorRetryAfterString() string {
+	return strconv.Itoa(errorRetryAfterSeconds + rand.Intn(errorRetryFuzzSeconds))
+}
+
+func notFoundRetryAfterString(misses int) string {
+	retryAfterS := notFoundRetryMinSeconds + notFoundRetryIncSeconds*misses
+	if retryAfterS > notFoundRetryMaxSeconds {
+		retryAfterS = notFoundRetryMaxSeconds
+	}
+	retryAfterS += rand.Intn(notFoundRetryFuzzSeconds)
+	return strconv.Itoa(retryAfterS)
+}
+
+func reannounceAfterString() string {
+	return strconv.Itoa(reannounceAfterSeconds + rand.Intn(reannounzeFuzzSeconds))
+}

cmd/stdiscosrv/apisrv_test.go

@@ -0,0 +1,65 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"net"
+	"testing"
+)
+
+func TestFixupAddresses(t *testing.T) {
+	cases := []struct {
+		remote net.IP
+		in     []string
+		out    []string
+	}{
+		{ // verbatim passthrough
+			in:  []string{"tcp://1.2.3.4:22000"},
+			out: []string{"tcp://1.2.3.4:22000"},
+		}, { // unspecified replaced by remote
+			remote: net.ParseIP("1.2.3.4"),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://1.2.3.4:22000", "tcp://192.0.2.42:22000"},
+		}, { // unspecified not used as replacement
+			remote: net.ParseIP("0.0.0.0"),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // unspecified not used as replacement
+			remote: net.ParseIP("::"),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // localhost not used as replacement
+			remote: net.ParseIP("127.0.0.1"),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // localhost not used as replacement
+			remote: net.ParseIP("::1"),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // multicast not used as replacement
+			remote: net.ParseIP("224.0.0.1"),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // multicast not used as replacement
+			remote: net.ParseIP("ff80::42"),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // explicitly announced weirdness is also filtered
+			remote: net.ParseIP("192.0.2.42"),
+			in:     []string{"tcp://:22000", "tcp://127.1.2.3:22000", "tcp://[::1]:22000", "tcp://[ff80::42]:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		},
+	}
+
+	for _, tc := range cases {
+		out := fixupAddresses(tc.remote, tc.in)
+		if fmt.Sprint(out) != fmt.Sprint(tc.out) {
+			t.Errorf("fixupAddresses(%v, %v) => %v, expected %v", tc.remote, tc.in, out, tc.out)
+		}
+	}
+}

cmd/stdiscosrv/database.go

@@ -0,0 +1,354 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//go:generate go run ../../script/protofmt.go database.proto
+//go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
+
+package main
+
+import (
+	"sort"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+type clock interface {
+	Now() time.Time
+}
+
+type defaultClock struct{}
+
+func (defaultClock) Now() time.Time {
+	return time.Now()
+}
+
+type database interface {
+	put(key string, rec DatabaseRecord) error
+	merge(key string, addrs []DatabaseAddress, seen int64) error
+	get(key string) (DatabaseRecord, error)
+}
+
+type levelDBStore struct {
+	db         *leveldb.DB
+	inbox      chan func()
+	stop       chan struct{}
+	clock      clock
+	marshalBuf []byte
+}
+
+func newLevelDBStore(dir string) (*levelDBStore, error) {
+	db, err := leveldb.OpenFile(dir, levelDBOptions)
+	if err != nil {
+		return nil, err
+	}
+	return &levelDBStore{
+		db:    db,
+		inbox: make(chan func(), 16),
+		stop:  make(chan struct{}),
+		clock: defaultClock{},
+	}, nil
+}
+
+func (s *levelDBStore) put(key string, rec DatabaseRecord) error {
+	t0 := time.Now()
+	defer func() {
+		databaseOperationSeconds.WithLabelValues(dbOpPut).Observe(time.Since(t0).Seconds())
+	}()
+
+	rc := make(chan error)
+
+	s.inbox <- func() {
+		size := rec.Size()
+		if len(s.marshalBuf) < size {
+			s.marshalBuf = make([]byte, size)
+		}
+		n, _ := rec.MarshalTo(s.marshalBuf)
+		rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
+	}
+
+	err := <-rc
+	if err != nil {
+		databaseOperations.WithLabelValues(dbOpPut, dbResError).Inc()
+	} else {
+		databaseOperations.WithLabelValues(dbOpPut, dbResSuccess).Inc()
+	}
+
+	return err
+}
+
+func (s *levelDBStore) merge(key string, addrs []DatabaseAddress, seen int64) error {
+	t0 := time.Now()
+	defer func() {
+		databaseOperationSeconds.WithLabelValues(dbOpMerge).Observe(time.Since(t0).Seconds())
+	}()
+
+	rc := make(chan error)
+	newRec := DatabaseRecord{
+		Addresses: addrs,
+		Seen:      seen,
+	}
+
+	s.inbox <- func() {
+		// grab the existing record
+		oldRec, err := s.get(key)
+		if err != nil {
+			// "not found" is not an error from get, so this is serious
+			// stuff only
+			rc <- err
+			return
+		}
+		newRec = merge(newRec, oldRec)
+
+		// We replicate s.put() functionality here ourselves instead of
+		// calling it because we want to serialize our get above together
+		// with the put in the same function.
+		size := newRec.Size()
+		if len(s.marshalBuf) < size {
+			s.marshalBuf = make([]byte, size)
+		}
+		n, _ := newRec.MarshalTo(s.marshalBuf)
+		rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
+	}
+
+	err := <-rc
+	if err != nil {
+		databaseOperations.WithLabelValues(dbOpMerge, dbResError).Inc()
+	} else {
+		databaseOperations.WithLabelValues(dbOpMerge, dbResSuccess).Inc()
+	}
+
+	return err
+}
+
+func (s *levelDBStore) get(key string) (DatabaseRecord, error) {
+	t0 := time.Now()
+	defer func() {
+		databaseOperationSeconds.WithLabelValues(dbOpGet).Observe(time.Since(t0).Seconds())
+	}()
+
+	keyBs := []byte(key)
+	val, err := s.db.Get(keyBs, nil)
+	if err == leveldb.ErrNotFound {
+		databaseOperations.WithLabelValues(dbOpGet, dbResNotFound).Inc()
+		return DatabaseRecord{}, nil
+	}
+	if err != nil {
+		databaseOperations.WithLabelValues(dbOpGet, dbResError).Inc()
+		return DatabaseRecord{}, err
+	}
+
+	var rec DatabaseRecord
+
+	if err := rec.Unmarshal(val); err != nil {
+		databaseOperations.WithLabelValues(dbOpGet, dbResUnmarshalError).Inc()
+		return DatabaseRecord{}, nil
+	}
+
+	rec.Addresses = expire(rec.Addresses, s.clock.Now().UnixNano())
+	databaseOperations.WithLabelValues(dbOpGet, dbResSuccess).Inc()
+	return rec, nil
+}
+
+func (s *levelDBStore) Serve() {
+	t := time.NewTimer(0)
+	defer t.Stop()
+	defer s.db.Close()
+
+	// Start the statistics serve routine. It will exit with us when
+	// statisticsTrigger is closed.
+	statisticsTrigger := make(chan struct{})
+	statisticsDone := make(chan struct{})
+	go s.statisticsServe(statisticsTrigger, statisticsDone)
+
+loop:
+	for {
+		select {
+		case fn := <-s.inbox:
+			// Run function in serialized order.
+			fn()
+
+		case <-t.C:
+			// Trigger the statistics routine to do its thing in the
+			// background.
+			statisticsTrigger <- struct{}{}
+
+		case <-statisticsDone:
+			// The statistics routine is done with one iteratation, schedule
+			// the next.
+			t.Reset(databaseStatisticsInterval)
+
+		case <-s.stop:
+			// We're done.
+			close(statisticsTrigger)
+			break loop
+		}
+	}
+
+	// Also wait for statisticsServe to return
+	<-statisticsDone
+}
+
+func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
+	defer close(done)
+
+	for range trigger {
+		t0 := time.Now()
+		nowNanos := t0.UnixNano()
+		cutoff24h := t0.Add(-24 * time.Hour).UnixNano()
+		cutoff1w := t0.Add(-7 * 24 * time.Hour).UnixNano()
+		cutoff2Mon := t0.Add(-60 * 24 * time.Hour).UnixNano()
+		current, last24h, last1w, inactive, errors := 0, 0, 0, 0, 0
+
+		iter := s.db.NewIterator(&util.Range{}, nil)
+		for iter.Next() {
+			// Attempt to unmarshal the record and count the
+			// failure if there's something wrong with it.
+			var rec DatabaseRecord
+			if err := rec.Unmarshal(iter.Value()); err != nil {
+				errors++
+				continue
+			}
+
+			// If there are addresses that have not expired it's a current
+			// record, otherwise account it based on when it was last seen
+			// (last 24 hours or last week) or finally as inactice.
+			switch {
+			case len(expire(rec.Addresses, nowNanos)) > 0:
+				current++
+			case rec.Seen > cutoff24h:
+				last24h++
+			case rec.Seen > cutoff1w:
+				last1w++
+			case rec.Seen > cutoff2Mon:
+				inactive++
+			case rec.Missed < cutoff2Mon:
+				// It hasn't been seen lately and we haven't recorded
+				// someone asking for this device in a long time either;
+				// delete the record.
+				if err := s.db.Delete(iter.Key(), nil); err != nil {
+					databaseOperations.WithLabelValues(dbOpDelete, dbResError).Inc()
+				} else {
+					databaseOperations.WithLabelValues(dbOpDelete, dbResSuccess).Inc()
+				}
+			default:
+				inactive++
+			}
+		}
+
+		iter.Release()
+
+		databaseKeys.WithLabelValues("current").Set(float64(current))
+		databaseKeys.WithLabelValues("last24h").Set(float64(last24h))
+		databaseKeys.WithLabelValues("last1w").Set(float64(last1w))
+		databaseKeys.WithLabelValues("inactive").Set(float64(inactive))
+		databaseKeys.WithLabelValues("error").Set(float64(errors))
+		databaseStatisticsSeconds.Set(time.Since(t0).Seconds())
+
+		// Signal that we are done and can be scheduled again.
+		done <- struct{}{}
+	}
+}
+
+func (s *levelDBStore) Stop() {
+	close(s.stop)
+}
+
+// merge returns the merged result of the two database records a and b. The
+// result is the union of the two address sets, with the newer expiry time
+// chosen for any duplicates.
+func merge(a, b DatabaseRecord) DatabaseRecord {
+	// Both lists must be sorted for this to work.
+	sort.Slice(a.Addresses, func(i, j int) bool {
+		return a.Addresses[i].Address < a.Addresses[j].Address
+	})
+	sort.Slice(b.Addresses, func(i, j int) bool {
+		return b.Addresses[i].Address < b.Addresses[j].Address
+	})
+
+	res := DatabaseRecord{
+		Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
+		Seen:      a.Seen,
+	}
+	if b.Seen > a.Seen {
+		res.Seen = b.Seen
+	}
+
+	aIdx := 0
+	bIdx := 0
+	aAddrs := a.Addresses
+	bAddrs := b.Addresses
+loop:
+	for {
+		switch {
+		case aIdx == len(aAddrs) && bIdx == len(bAddrs):
+			// both lists are exhausted, we are done
+			break loop
+
+		case aIdx == len(aAddrs):
+			// a is exhausted, pick from b and continue
+			res.Addresses = append(res.Addresses, bAddrs[bIdx])
+			bIdx++
+			continue
+
+		case bIdx == len(bAddrs):
+			// b is exhausted, pick from a and continue
+			res.Addresses = append(res.Addresses, aAddrs[aIdx])
+			aIdx++
+			continue
+		}
+
+		// We have values left on both sides.
+		aVal := aAddrs[aIdx]
+		bVal := bAddrs[bIdx]
+
+		switch {
+		case aVal.Address == bVal.Address:
+			// update for same address, pick newer
+			if aVal.Expires > bVal.Expires {
+				res.Addresses = append(res.Addresses, aVal)
+			} else {
+				res.Addresses = append(res.Addresses, bVal)
+			}
+			aIdx++
+			bIdx++
+
+		case aVal.Address < bVal.Address:
+			// a is smallest, pick it and continue
+			res.Addresses = append(res.Addresses, aVal)
+			aIdx++
+
+		default:
+			// b is smallest, pick it and continue
+			res.Addresses = append(res.Addresses, bVal)
+			bIdx++
+		}
+	}
+	return res
+}
+
+// expire returns the list of addresses after removing expired entries.
+// Expiration happen in place, so the slice given as the parameter is
+// destroyed. Internal order is not preserved.
+func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
+	i := 0
+	for i < len(addrs) {
+		if addrs[i].Expires < now {
+			// This item is expired. Replace it with the last in the list
+			// (noop if we are at the last item).
+			addrs[i] = addrs[len(addrs)-1]
+			// Wipe the last item of the list to release references to
+			// strings and stuff.
+			addrs[len(addrs)-1] = DatabaseAddress{}
+			// Shorten the slice.
+			addrs = addrs[:len(addrs)-1]
+			continue
+		}
+		i++
+	}
+	return addrs
+}

cmd/stdiscosrv/database.pb.go

@@ -0,0 +1,882 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: database.proto
+
+package main
+
+import (
+	fmt "fmt"
+	_ "github.com/gogo/protobuf/gogoproto"
+	proto "github.com/gogo/protobuf/proto"
+	io "io"
+	math "math"
+	math_bits "math/bits"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+
+type DatabaseRecord struct {
+	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
+	Misses    int32             `protobuf:"varint,2,opt,name=misses,proto3" json:"misses,omitempty"`
+	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
+	Missed    int64             `protobuf:"varint,4,opt,name=missed,proto3" json:"missed,omitempty"`
+}
+
+func (m *DatabaseRecord) Reset()         { *m = DatabaseRecord{} }
+func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
+func (*DatabaseRecord) ProtoMessage()    {}
+func (*DatabaseRecord) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{0}
+}
+func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseRecord.Merge(m, src)
+}
+func (m *DatabaseRecord) XXX_Size() int {
+	return m.Size()
+}
+func (m *DatabaseRecord) XXX_DiscardUnknown() {
+	xxx_messageInfo_DatabaseRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DatabaseRecord proto.InternalMessageInfo
+
+type ReplicationRecord struct {
+	Key       string            `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+	Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses"`
+	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
+}
+
+func (m *ReplicationRecord) Reset()         { *m = ReplicationRecord{} }
+func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
+func (*ReplicationRecord) ProtoMessage()    {}
+func (*ReplicationRecord) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{1}
+}
+func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ReplicationRecord.Merge(m, src)
+}
+func (m *ReplicationRecord) XXX_Size() int {
+	return m.Size()
+}
+func (m *ReplicationRecord) XXX_DiscardUnknown() {
+	xxx_messageInfo_ReplicationRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ReplicationRecord proto.InternalMessageInfo
+
+type DatabaseAddress struct {
+	Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
+	Expires int64  `protobuf:"varint,2,opt,name=expires,proto3" json:"expires,omitempty"`
+}
+
+func (m *DatabaseAddress) Reset()         { *m = DatabaseAddress{} }
+func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
+func (*DatabaseAddress) ProtoMessage()    {}
+func (*DatabaseAddress) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{2}
+}
+func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseAddress.Merge(m, src)
+}
+func (m *DatabaseAddress) XXX_Size() int {
+	return m.Size()
+}
+func (m *DatabaseAddress) XXX_DiscardUnknown() {
+	xxx_messageInfo_DatabaseAddress.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DatabaseAddress proto.InternalMessageInfo
+
+func init() {
+	proto.RegisterType((*DatabaseRecord)(nil), "main.DatabaseRecord")
+	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
+	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
+}
+
+func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
+
+var fileDescriptor_b90fe3356ea5df07 = []byte{
+	// 270 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
+	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
+	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
+	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
+	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
+	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
+	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
+	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
+	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
+	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
+	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
+	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
+	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
+	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
+	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
+	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
+	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
+}
+
+func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Missed != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+		i--
+		dAtA[i] = 0x20
+	}
+	if m.Seen != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
+	}
+	if m.Misses != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+		i--
+		dAtA[i] = 0x10
+	}
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Seen != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
+	}
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x12
+		}
+	}
+	if len(m.Key) > 0 {
+		i -= len(m.Key)
+		copy(dAtA[i:], m.Key)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Expires != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
+		i--
+		dAtA[i] = 0x10
+	}
+	if len(m.Address) > 0 {
+		i -= len(m.Address)
+		copy(dAtA[i:], m.Address)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
+	offset -= sovDatabase(v)
+	base := offset
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return base
+}
+func (m *DatabaseRecord) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Addresses) > 0 {
+		for _, e := range m.Addresses {
+			l = e.Size()
+			n += 1 + l + sovDatabase(uint64(l))
+		}
+	}
+	if m.Misses != 0 {
+		n += 1 + sovDatabase(uint64(m.Misses))
+	}
+	if m.Seen != 0 {
+		n += 1 + sovDatabase(uint64(m.Seen))
+	}
+	if m.Missed != 0 {
+		n += 1 + sovDatabase(uint64(m.Missed))
+	}
+	return n
+}
+
+func (m *ReplicationRecord) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Key)
+	if l > 0 {
+		n += 1 + l + sovDatabase(uint64(l))
+	}
+	if len(m.Addresses) > 0 {
+		for _, e := range m.Addresses {
+			l = e.Size()
+			n += 1 + l + sovDatabase(uint64(l))
+		}
+	}
+	if m.Seen != 0 {
+		n += 1 + sovDatabase(uint64(m.Seen))
+	}
+	return n
+}
+
+func (m *DatabaseAddress) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Address)
+	if l > 0 {
+		n += 1 + l + sovDatabase(uint64(l))
+	}
+	if m.Expires != 0 {
+		n += 1 + sovDatabase(uint64(m.Expires))
+	}
+	return n
+}
+
+func sovDatabase(x uint64) (n int) {
+	return (math_bits.Len64(x|1) + 6) / 7
+}
+func sozDatabase(x uint64) (n int) {
+	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: DatabaseRecord: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: DatabaseRecord: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Addresses = append(m.Addresses, DatabaseAddress{})
+			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Misses", wireType)
+			}
+			m.Misses = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Misses |= int32(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
+			}
+			m.Seen = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Seen |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Missed", wireType)
+			}
+			m.Missed = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Missed |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ReplicationRecord: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ReplicationRecord: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Key = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Addresses = append(m.Addresses, DatabaseAddress{})
+			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
+			}
+			m.Seen = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Seen |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: DatabaseAddress: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: DatabaseAddress: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Address", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Address = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Expires", wireType)
+			}
+			m.Expires = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Expires |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipDatabase(dAtA []byte) (n int, err error) {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if dAtA[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+			return iNdEx, nil
+		case 1:
+			iNdEx += 8
+			return iNdEx, nil
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if length < 0 {
+				return 0, ErrInvalidLengthDatabase
+			}
+			iNdEx += length
+			if iNdEx < 0 {
+				return 0, ErrInvalidLengthDatabase
+			}
+			return iNdEx, nil
+		case 3:
+			for {
+				var innerWire uint64
+				var start int = iNdEx
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return 0, ErrIntOverflowDatabase
+					}
+					if iNdEx >= l {
+						return 0, io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					innerWire |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				innerWireType := int(innerWire & 0x7)
+				if innerWireType == 4 {
+					break
+				}
+				next, err := skipDatabase(dAtA[start:])
+				if err != nil {
+					return 0, err
+				}
+				iNdEx = start + next
+				if iNdEx < 0 {
+					return 0, ErrInvalidLengthDatabase
+				}
+			}
+			return iNdEx, nil
+		case 4:
+			return iNdEx, nil
+		case 5:
+			iNdEx += 4
+			return iNdEx, nil
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+	}
+	panic("unreachable")
+}
+
+var (
+	ErrInvalidLengthDatabase = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowDatabase   = fmt.Errorf("proto: integer overflow")
+)

cmd/stdiscosrv/database.proto

@@ -0,0 +1,36 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+syntax = "proto3";
+
+package main;
+
+import "repos/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.goproto_unkeyed_all) = false;
+option (gogoproto.goproto_unrecognized_all) = false;
+option (gogoproto.goproto_sizecache_all) = false;
+
+message DatabaseRecord {
+    repeated DatabaseAddress addresses = 1 [(gogoproto.nullable) = false];
+    int32                    misses    = 2; // Number of lookups* without hits
+    int64                    seen      = 3; // Unix nanos, last device announce
+    int64                    missed    = 4; // Unix nanos, last* failed lookup
+}
+
+// *) Not every lookup results in a write, so may not be completely accurate
+
+message ReplicationRecord {
+    string                   key       = 1;
+    repeated DatabaseAddress addresses = 2 [(gogoproto.nullable) = false];
+    int64                    seen      = 3; // Unix nanos, last device announce
+}
+
+message DatabaseAddress {
+    string address = 1;
+    int64  expires = 2; // Unix nanos
+}

cmd/stdiscosrv/database_test.go

@@ -0,0 +1,211 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestDatabaseGetSet(t *testing.T) {
+	os.RemoveAll("_database")
+	defer os.RemoveAll("_database")
+	db, err := newLevelDBStore("_database")
+	if err != nil {
+		t.Fatal(err)
+	}
+	go db.Serve()
+	defer db.Stop()
+
+	// Check missing record
+
+	rec, err := db.get("abcd")
+	if err != nil {
+		t.Error("not found should not be an error")
+	}
+	if len(rec.Addresses) != 0 {
+		t.Error("addresses should be empty")
+	}
+	if rec.Misses != 0 {
+		t.Error("missing should be zero")
+	}
+
+	// Set up a clock
+
+	now := time.Now()
+	tc := &testClock{now}
+	db.clock = tc
+
+	// Put a record
+
+	rec.Addresses = []DatabaseAddress{
+		{Address: "tcp://1.2.3.4:5", Expires: tc.Now().Add(time.Minute).UnixNano()},
+	}
+	if err := db.put("abcd", rec); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it
+
+	rec, err = db.get("abcd")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 1 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have one address")
+	}
+	if rec.Addresses[0].Address != "tcp://1.2.3.4:5" {
+		t.Log(rec.Addresses)
+		t.Error("incorrect address")
+	}
+
+	// Wind the clock one half expiry, and merge in a new address
+
+	tc.wind(30 * time.Second)
+
+	addrs := []DatabaseAddress{
+		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
+	}
+	if err := db.merge("abcd", addrs, tc.Now().UnixNano()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it
+
+	rec, err = db.get("abcd")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 2 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have two addresses")
+	}
+	if rec.Addresses[0].Address != "tcp://1.2.3.4:5" {
+		t.Log(rec.Addresses)
+		t.Error("incorrect address[0]")
+	}
+	if rec.Addresses[1].Address != "tcp://6.7.8.9:0" {
+		t.Log(rec.Addresses)
+		t.Error("incorrect address[1]")
+	}
+
+	// Pass the first expiry time
+
+	tc.wind(45 * time.Second)
+
+	// Verify it
+
+	rec, err = db.get("abcd")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 1 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have one address")
+	}
+	if rec.Addresses[0].Address != "tcp://6.7.8.9:0" {
+		t.Log(rec.Addresses)
+		t.Error("incorrect address")
+	}
+
+	// Put a record with misses
+
+	rec = DatabaseRecord{Misses: 42}
+	if err := db.put("efgh", rec); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it
+
+	rec, err = db.get("efgh")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 0 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have no addresses")
+	}
+	if rec.Misses != 42 {
+		t.Log(rec.Misses)
+		t.Error("incorrect misses")
+	}
+
+	// Set an address
+
+	addrs = []DatabaseAddress{
+		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
+	}
+	if err := db.merge("efgh", addrs, tc.Now().UnixNano()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it
+
+	rec, err = db.get("efgh")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 1 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have one address")
+	}
+	if rec.Misses != 0 {
+		t.Log(rec.Misses)
+		t.Error("should have no misses")
+	}
+}
+
+func TestFilter(t *testing.T) {
+	// all cases are expired with t=10
+	cases := []struct {
+		a []DatabaseAddress
+		b []DatabaseAddress
+	}{
+		{
+			a: nil,
+			b: nil,
+		},
+		{
+			a: []DatabaseAddress{{Address: "a", Expires: 9}, {Address: "b", Expires: 9}, {Address: "c", Expires: 9}},
+			b: []DatabaseAddress{},
+		},
+		{
+			a: []DatabaseAddress{{Address: "a", Expires: 10}},
+			b: []DatabaseAddress{{Address: "a", Expires: 10}},
+		},
+		{
+			a: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
+			b: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
+		},
+		{
+			a: []DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
+			b: []DatabaseAddress{{Address: "d", Expires: 15}, {Address: "b", Expires: 15}}, // gets reordered
+		},
+	}
+
+	for _, tc := range cases {
+		res := expire(tc.a, 10)
+		if fmt.Sprint(res) != fmt.Sprint(tc.b) {
+			t.Errorf("Incorrect result %v, expected %v", res, tc.b)
+		}
+	}
+}
+
+type testClock struct {
+	now time.Time
+}
+
+func (t *testClock) wind(d time.Duration) {
+	t.now = t.now.Add(d)
+}
+
+func (t *testClock) Now() time.Time {
+	return t.now
+}

cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv

@@ -0,0 +1,4 @@
+[stdiscosrv]
+title=Syncthing discovery server
+description=Lets syncthing clients discover each other
+ports=8443/tcp

cmd/stdiscosrv/etc/linux-systemd/default

@@ -0,0 +1,3 @@
+# Default settings for syncthing-relaysrv (strelaysrv).
+## Add Options here:
+DISCOSRV_OPTS=

cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=Syncthing Discovery Server
+After=network.target
+Documentation=man:stdiscosrv(1)
+
+[Service]
+WorkingDirectory=/var/lib/syncthing-discosrv
+EnvironmentFile=/etc/default/syncthing-discosrv
+ExecStart=/usr/bin/stdiscosrv $DISCOSRV_OPTS
+
+# Hardening
+User=syncthing-discosrv
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-discosrv
+NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
+
+[Install]
+WantedBy=multi-user.target
+Alias=syncthing-discosrv.service

cmd/stdiscosrv/main.go

@@ -0,0 +1,186 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"crypto/tls"
+	"flag"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/tlsutil"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/thejerf/suture"
+)
+
+const (
+	addressExpiryTime          = 2 * time.Hour
+	databaseStatisticsInterval = 5 * time.Minute
+
+	// Reannounce-After is set to reannounceAfterSeconds +
+	// random(reannounzeFuzzSeconds), similar for Retry-After
+	reannounceAfterSeconds = 3300
+	reannounzeFuzzSeconds  = 300
+	errorRetryAfterSeconds = 1500
+	errorRetryFuzzSeconds  = 300
+
+	// Retry for not found is minSeconds + failures * incSeconds +
+	// random(fuzz), where failures is the number of consecutive lookups
+	// with no answer, up to maxSeconds. The fuzz is applied after capping
+	// to maxSeconds.
+	notFoundRetryMinSeconds  = 60
+	notFoundRetryMaxSeconds  = 3540
+	notFoundRetryIncSeconds  = 10
+	notFoundRetryFuzzSeconds = 60
+
+	// How often (in requests) we serialize the missed counter to database.
+	notFoundMissesWriteInterval = 10
+
+	httpReadTimeout    = 5 * time.Second
+	httpWriteTimeout   = 5 * time.Second
+	httpMaxHeaderBytes = 1 << 10
+
+	// Size of the replication outbox channel
+	replicationOutboxSize = 10000
+)
+
+// These options make the database a little more optimized for writes, at
+// the expense of some memory usage and risk of losing writes in a (system)
+// crash.
+var levelDBOptions = &opt.Options{
+	NoSync:      true,
+	WriteBuffer: 32 << 20, // default 4<<20
+}
+
+var (
+	debug = false
+)
+
+func main() {
+	var listen string
+	var dir string
+	var metricsListen string
+	var replicationListen string
+	var replicationPeers string
+	var certFile string
+	var keyFile string
+	var useHTTP bool
+
+	log.SetOutput(os.Stdout)
+	log.SetFlags(0)
+
+	flag.StringVar(&certFile, "cert", "./cert.pem", "Certificate file")
+	flag.StringVar(&dir, "db-dir", "./discovery.db", "Database directory")
+	flag.BoolVar(&debug, "debug", false, "Print debug output")
+	flag.BoolVar(&useHTTP, "http", false, "Listen on HTTP (behind an HTTPS proxy)")
+	flag.StringVar(&listen, "listen", ":8443", "Listen address")
+	flag.StringVar(&keyFile, "key", "./key.pem", "Key file")
+	flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
+	flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
+	flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
+	showVersion := flag.Bool("version", false, "Show version")
+	flag.Parse()
+
+	log.Println(build.LongVersion)
+	if *showVersion {
+		return
+	}
+
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		log.Println("Failed to load keypair. Generating one, this might take a while...")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv")
+		if err != nil {
+			log.Fatalln("Failed to generate X509 key pair:", err)
+		}
+	}
+
+	devID := protocol.NewDeviceID(cert.Certificate[0])
+	log.Println("Server device ID is", devID)
+
+	// Parse the replication specs, if any.
+	var allowedReplicationPeers []protocol.DeviceID
+	var replicationDestinations []string
+	parts := strings.Split(replicationPeers, ",")
+	for _, part := range parts {
+		fields := strings.Split(part, "@")
+
+		switch len(fields) {
+		case 2:
+			// This is an id@address specification. Grab the address for the
+			// destination list. Try to resolve it once to catch obvious
+			// syntax errors here rather than having the sender service fail
+			// repeatedly later.
+			_, err := net.ResolveTCPAddr("tcp", fields[1])
+			if err != nil {
+				log.Fatalln("Resolving address:", err)
+			}
+			replicationDestinations = append(replicationDestinations, fields[1])
+			fallthrough // N.B.
+
+		case 1:
+			// The first part is always a device ID.
+			id, err := protocol.DeviceIDFromString(fields[0])
+			if err != nil {
+				log.Fatalln("Parsing device ID:", err)
+			}
+			allowedReplicationPeers = append(allowedReplicationPeers, id)
+
+		default:
+			log.Fatalln("Unrecognized replication spec:", part)
+		}
+	}
+
+	// Root of the service tree.
+	main := suture.New("main", suture.Spec{
+		PassThroughPanics: true,
+	})
+
+	// Start the database.
+	db, err := newLevelDBStore(dir)
+	if err != nil {
+		log.Fatalln("Open database:", err)
+	}
+	main.Add(db)
+
+	// Start any replication senders.
+	var repl replicationMultiplexer
+	for _, dst := range replicationDestinations {
+		rs := newReplicationSender(dst, cert, allowedReplicationPeers)
+		main.Add(rs)
+		repl = append(repl, rs)
+	}
+
+	// If we have replication configured, start the replication listener.
+	if len(allowedReplicationPeers) > 0 {
+		rl := newReplicationListener(replicationListen, cert, allowedReplicationPeers, db)
+		main.Add(rl)
+	}
+
+	// Start the main API server.
+	qs := newAPISrv(listen, cert, db, repl, useHTTP)
+	main.Add(qs)
+
+	// If we have a metrics port configured, start a metrics handler.
+	if metricsListen != "" {
+		go func() {
+			mux := http.NewServeMux()
+			mux.Handle("/metrics", promhttp.Handler())
+			log.Fatal(http.ListenAndServe(metricsListen, mux))
+		}()
+	}
+
+	// Engage!
+	main.Serve()
+}

cmd/stdiscosrv/replication.go

@@ -0,0 +1,326 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"crypto/tls"
+	"encoding/binary"
+	"fmt"
+	io "io"
+	"log"
+	"net"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+const replicationReadTimeout = time.Minute
+const replicationHeartbeatInterval = time.Second * 30
+
+type replicator interface {
+	send(key string, addrs []DatabaseAddress, seen int64)
+}
+
+// a replicationSender tries to connect to the remote address and provide
+// them with a feed of replication updates.
+type replicationSender struct {
+	dst        string
+	cert       tls.Certificate // our certificate
+	allowedIDs []protocol.DeviceID
+	outbox     chan ReplicationRecord
+	stop       chan struct{}
+}
+
+func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
+	return &replicationSender{
+		dst:        dst,
+		cert:       cert,
+		allowedIDs: allowedIDs,
+		outbox:     make(chan ReplicationRecord, replicationOutboxSize),
+		stop:       make(chan struct{}),
+	}
+}
+
+func (s *replicationSender) Serve() {
+	// Sleep a little at startup. Peers often restart at the same time, and
+	// this avoid the service failing and entering backoff state
+	// unnecessarily, while also reducing the reconnect rate to something
+	// reasonable by default.
+	time.Sleep(2 * time.Second)
+
+	tlsCfg := &tls.Config{
+		Certificates:       []tls.Certificate{s.cert},
+		MinVersion:         tls.VersionTLS12,
+		InsecureSkipVerify: true,
+	}
+
+	// Dial the TLS connection.
+	conn, err := tls.Dial("tcp", s.dst, tlsCfg)
+	if err != nil {
+		log.Println("Replication connect:", err)
+		return
+	}
+	defer func() {
+		conn.SetWriteDeadline(time.Now().Add(time.Second))
+		conn.Close()
+	}()
+
+	// Get the other side device ID.
+	remoteID, err := deviceID(conn)
+	if err != nil {
+		log.Println("Replication connect:", err)
+		return
+	}
+
+	// Verify it's in the set of allowed device IDs.
+	if !deviceIDIn(remoteID, s.allowedIDs) {
+		log.Println("Replication connect: unexpected device ID:", remoteID)
+		return
+	}
+
+	heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
+	defer heartBeatTicker.Stop()
+
+	// Send records.
+	buf := make([]byte, 1024)
+	for {
+		select {
+		case <-heartBeatTicker.C:
+			if len(s.outbox) > 0 {
+				// No need to send heartbeats if there are events/prevrious
+				// heartbeats to send, they will keep the connection alive.
+				continue
+			}
+			// Empty replication message is the heartbeat:
+			s.outbox <- ReplicationRecord{}
+
+		case rec := <-s.outbox:
+			// Buffer must hold record plus four bytes for size
+			size := rec.Size()
+			if len(buf) < size+4 {
+				buf = make([]byte, size+4)
+			}
+
+			// Record comes after the four bytes size
+			n, err := rec.MarshalTo(buf[4:])
+			if err != nil {
+				// odd to get an error here, but we haven't sent anything
+				// yet so it's not fatal
+				replicationSendsTotal.WithLabelValues("error").Inc()
+				log.Println("Replication marshal:", err)
+				continue
+			}
+			binary.BigEndian.PutUint32(buf, uint32(n))
+
+			// Send
+			conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
+			if _, err := conn.Write(buf[:4+n]); err != nil {
+				replicationSendsTotal.WithLabelValues("error").Inc()
+				log.Println("Replication write:", err)
+				// Yes, we are loosing the replication event here.
+				return
+			}
+			replicationSendsTotal.WithLabelValues("success").Inc()
+
+		case <-s.stop:
+			return
+		}
+	}
+}
+
+func (s *replicationSender) Stop() {
+	close(s.stop)
+}
+
+func (s *replicationSender) String() string {
+	return fmt.Sprintf("replicationSender(%q)", s.dst)
+}
+
+func (s *replicationSender) send(key string, ps []DatabaseAddress, seen int64) {
+	item := ReplicationRecord{
+		Key:       key,
+		Addresses: ps,
+	}
+
+	// The send should never block. The inbox is suitably buffered for at
+	// least a few seconds of stalls, which shouldn't happen in practice.
+	select {
+	case s.outbox <- item:
+	default:
+		replicationSendsTotal.WithLabelValues("drop").Inc()
+	}
+}
+
+// a replicationMultiplexer sends to multiple replicators
+type replicationMultiplexer []replicator
+
+func (m replicationMultiplexer) send(key string, ps []DatabaseAddress, seen int64) {
+	for _, s := range m {
+		// each send is nonblocking
+		s.send(key, ps, seen)
+	}
+}
+
+// replicationListener accepts incoming connections and reads replication
+// items from them. Incoming items are applied to the KV store.
+type replicationListener struct {
+	addr       string
+	cert       tls.Certificate
+	allowedIDs []protocol.DeviceID
+	db         database
+	stop       chan struct{}
+}
+
+func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
+	return &replicationListener{
+		addr:       addr,
+		cert:       cert,
+		allowedIDs: allowedIDs,
+		db:         db,
+		stop:       make(chan struct{}),
+	}
+}
+
+func (l *replicationListener) Serve() {
+	tlsCfg := &tls.Config{
+		Certificates:       []tls.Certificate{l.cert},
+		ClientAuth:         tls.RequestClientCert,
+		MinVersion:         tls.VersionTLS12,
+		InsecureSkipVerify: true,
+	}
+
+	lst, err := tls.Listen("tcp", l.addr, tlsCfg)
+	if err != nil {
+		log.Println("Replication listen:", err)
+		return
+	}
+	defer lst.Close()
+
+	for {
+		select {
+		case <-l.stop:
+			return
+		default:
+		}
+
+		// Accept a connection
+		conn, err := lst.Accept()
+		if err != nil {
+			log.Println("Replication accept:", err)
+			return
+		}
+
+		// Figure out the other side device ID
+		remoteID, err := deviceID(conn.(*tls.Conn))
+		if err != nil {
+			log.Println("Replication accept:", err)
+			conn.SetWriteDeadline(time.Now().Add(time.Second))
+			conn.Close()
+			continue
+		}
+
+		// Verify it is in the set of allowed device IDs
+		if !deviceIDIn(remoteID, l.allowedIDs) {
+			log.Println("Replication accept: unexpected device ID:", remoteID)
+			conn.SetWriteDeadline(time.Now().Add(time.Second))
+			conn.Close()
+			continue
+		}
+
+		go l.handle(conn)
+	}
+}
+
+func (l *replicationListener) Stop() {
+	close(l.stop)
+}
+
+func (l *replicationListener) String() string {
+	return fmt.Sprintf("replicationListener(%q)", l.addr)
+}
+
+func (l *replicationListener) handle(conn net.Conn) {
+	defer func() {
+		conn.SetWriteDeadline(time.Now().Add(time.Second))
+		conn.Close()
+	}()
+
+	buf := make([]byte, 1024)
+
+	for {
+		select {
+		case <-l.stop:
+			return
+		default:
+		}
+
+		conn.SetReadDeadline(time.Now().Add(replicationReadTimeout))
+
+		// First four bytes are the size
+		if _, err := io.ReadFull(conn, buf[:4]); err != nil {
+			log.Println("Replication read size:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			return
+		}
+
+		// Read the rest of the record
+		size := int(binary.BigEndian.Uint32(buf[:4]))
+		if len(buf) < size {
+			buf = make([]byte, size)
+		}
+
+		if size == 0 {
+			// Heartbeat, ignore
+			continue
+		}
+
+		if _, err := io.ReadFull(conn, buf[:size]); err != nil {
+			log.Println("Replication read record:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			return
+		}
+
+		// Unmarshal
+		var rec ReplicationRecord
+		if err := rec.Unmarshal(buf[:size]); err != nil {
+			log.Println("Replication unmarshal:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			continue
+		}
+
+		// Store
+		l.db.merge(rec.Key, rec.Addresses, rec.Seen)
+		replicationRecvsTotal.WithLabelValues("success").Inc()
+	}
+}
+
+func deviceID(conn *tls.Conn) (protocol.DeviceID, error) {
+	// Handshake may not be complete on the server side yet, which we need
+	// to get the client certificate.
+	if !conn.ConnectionState().HandshakeComplete {
+		if err := conn.Handshake(); err != nil {
+			return protocol.DeviceID{}, err
+		}
+	}
+
+	// We expect exactly one certificate.
+	certs := conn.ConnectionState().PeerCertificates
+	if len(certs) != 1 {
+		return protocol.DeviceID{}, fmt.Errorf("unexpected number of certificates (%d != 1)", len(certs))
+	}
+
+	return protocol.NewDeviceID(certs[0].Raw), nil
+}
+
+func deviceIDIn(id protocol.DeviceID, ids []protocol.DeviceID) bool {
+	for _, candidate := range ids {
+		if id == candidate {
+			return true
+		}
+	}
+	return false
+}

cmd/stdiscosrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-discosrv --ingroup syncthing syncthing-discosrv

cmd/stdiscosrv/stats.go

@@ -0,0 +1,123 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"os"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	apiRequestsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "api_requests_total",
+			Help:      "Number of API requests.",
+		}, []string{"type", "result"})
+	apiRequestsSeconds = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Namespace:  "syncthing",
+			Subsystem:  "discovery",
+			Name:       "api_requests_seconds",
+			Help:       "Latency of API requests.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		}, []string{"type"})
+
+	lookupRequestsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "lookup_requests_total",
+			Help:      "Number of lookup requests.",
+		}, []string{"result"})
+	announceRequestsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "announcement_requests_total",
+			Help:      "Number of announcement requests.",
+		}, []string{"result"})
+
+	replicationSendsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "replication_sends_total",
+			Help:      "Number of replication sends.",
+		}, []string{"result"})
+	replicationRecvsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "replication_recvs_total",
+			Help:      "Number of replication receives.",
+		}, []string{"result"})
+
+	databaseKeys = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "database_keys",
+			Help:      "Number of database keys at last count.",
+		}, []string{"category"})
+	databaseStatisticsSeconds = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "database_statistics_seconds",
+			Help:      "Time spent running the statistics routine.",
+		})
+
+	databaseOperations = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "database_operations_total",
+			Help:      "Number of database operations.",
+		}, []string{"operation", "result"})
+	databaseOperationSeconds = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Namespace:  "syncthing",
+			Subsystem:  "discovery",
+			Name:       "database_operation_seconds",
+			Help:       "Latency of database operations.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		}, []string{"operation"})
+)
+
+const (
+	dbOpGet             = "get"
+	dbOpPut             = "put"
+	dbOpMerge           = "merge"
+	dbOpDelete          = "delete"
+	dbResSuccess        = "success"
+	dbResNotFound       = "not_found"
+	dbResError          = "error"
+	dbResUnmarshalError = "unmarsh_err"
+)
+
+func init() {
+	prometheus.MustRegister(apiRequestsTotal, apiRequestsSeconds,
+		lookupRequestsTotal, announceRequestsTotal,
+		replicationSendsTotal, replicationRecvsTotal,
+		databaseKeys, databaseStatisticsSeconds,
+		databaseOperations, databaseOperationSeconds)
+
+	processCollectorOpts := prometheus.ProcessCollectorOpts{
+		Namespace: "syncthing_discovery",
+		PidFn: func() (int, error) {
+			return os.Getpid(), nil
+		},
+	}
+
+	prometheus.MustRegister(
+		prometheus.NewProcessCollector(processCollectorOpts),
+	)
+
+}

cmd/stevents/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/stfileinfo/main.go

@@ -2,11 +2,12 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
+	"context"
 	"flag"
 	"log"
 	"os"
@@ -40,7 +41,8 @@ func main() {
 	log.Println("Lstat:")
 	log.Printf("  Size: %d bytes", fi.Size())
 	log.Printf("  Mode: 0%o", fi.Mode())
-	log.Printf("  Time: %v (%d)", fi.ModTime(), fi.ModTime().Unix())
+	log.Printf("  Time: %v", fi.ModTime())
+	log.Printf("        %d.%09d", fi.ModTime().Unix(), fi.ModTime().Nanosecond())
 	log.Println()
 
 	if !fi.Mode().IsDir() && !fi.Mode().IsRegular() {
@@ -52,7 +54,8 @@ func main() {
 		log.Println("Stat:")
 		log.Printf("  Size: %d bytes", fi.Size())
 		log.Printf("  Mode: 0%o", fi.Mode())
-		log.Printf("  Time: %v (%d)", fi.ModTime(), fi.ModTime().Unix())
+		log.Printf("  Time: %v", fi.ModTime())
+		log.Printf("        %d.%09d", fi.ModTime().Unix(), fi.ModTime().Nanosecond())
 		log.Println()
 	}
 
@@ -65,10 +68,10 @@ func main() {
 		}
 
 		blockSize := int(fi.Size())
-		if *standardBlocks || blockSize < protocol.BlockSize {
-			blockSize = protocol.BlockSize
+		if *standardBlocks || blockSize < protocol.MinBlockSize {
+			blockSize = protocol.BlockSize(fi.Size())
 		}
-		bs, err := scanner.Blocks(fd, blockSize, fi.Size(), nil)
+		bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/stfinddevice/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -17,6 +17,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -66,7 +67,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 		}()
 	}
 
-	for _ = range servers {
+	for range servers {
 		res := <-resc
 
 		u, _ := url.Parse(res.server)
@@ -82,7 +83,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 }
 
 func checkServer(deviceID protocol.DeviceID, server string) checkResult {
-	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger)
 	if err != nil {
 		return checkResult{error: err}
 	}

cmd/stfindignored/main.go

@@ -0,0 +1,44 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Commmand stfindignored lists ignored files under a given folder root.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/syncthing/syncthing/lib/fs"
+	"github.com/syncthing/syncthing/lib/ignore"
+)
+
+func main() {
+	flag.Parse()
+	root := flag.Arg(0)
+	if root == "" {
+		root = "."
+	}
+
+	vfs := fs.NewWalkFilesystem(fs.NewFilesystem(fs.FilesystemTypeBasic, root))
+
+	ign := ignore.New(vfs)
+	if err := ign.Load(".stignore"); err != nil {
+		fmt.Fprintf(os.Stderr, "Fatal: loading ignores: %v\n", err)
+		os.Exit(1)
+	}
+
+	vfs.Walk(".", func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Warning: %s: %v\n", path, err)
+			return fs.SkipDir
+		}
+		if ign.Match(path).IsIgnored() {
+			fmt.Println(path)
+		}
+		return nil
+	})
+}

cmd/stgenfiles/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -66,7 +66,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
 }
 
 func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
-	src := io.LimitReader(&inifiteReader{fd}, int64(s))
+	src := io.LimitReader(&inifiteReader{fd}, s)
 	dst, err := os.Create(p1)
 	if err != nil {
 		return err
@@ -82,15 +82,10 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
 		return err
 	}
 
-	_ = os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
+	os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
 
 	t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
-	err = os.Chtimes(p1, t, t)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return os.Chtimes(p1, t, t)
 }
 
 func randomName() string {

cmd/stindex/dump.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -10,51 +10,71 @@ import (
 	"encoding/binary"
 	"fmt"
 	"log"
+	"time"
 
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syndtr/goleveldb/leveldb"
 )
 
-func dump(ldb *leveldb.DB) {
+func dump(ldb *db.Lowlevel) {
 	it := ldb.NewIterator(nil, nil)
-	var dev protocol.DeviceID
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {
 		case db.KeyTypeDevice:
-			folder := nulString(key[1 : 1+64])
-			devBytes := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			copy(dev[:], devBytes)
-			fmt.Printf("[device] F:%q N:%q D:%v\n", folder, name, dev)
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+			fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
 
 			var f protocol.FileInfo
-			err := f.UnmarshalXDR(it.Value())
+			err := f.Unmarshal(it.Value())
 			if err != nil {
 				log.Fatal(err)
 			}
-			fmt.Printf("  N:%q\n  F:%#o\n  M:%d\n  V:%v\n  S:%d\n  B:%d\n", f.Name, f.Flags, f.Modified, f.Version, f.Size(), len(f.Blocks))
+			fmt.Printf(" V:%v\n", f)
 
 		case db.KeyTypeGlobal:
-			folder := nulString(key[1 : 1+64])
-			name := nulString(key[1+64:])
-			fmt.Printf("[global] F:%q N:%q V:%x\n", folder, name, it.Value())
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			var flv db.VersionList
+			flv.Unmarshal(it.Value())
+			fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
 
 		case db.KeyTypeBlock:
-			folder := nulString(key[1 : 1+64])
-			hash := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			fmt.Printf("[block] F:%q H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[1+4 : 1+4+32]
+			name := nulString(key[1+4+32:])
+			fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
 
 		case db.KeyTypeDeviceStatistic:
-			fmt.Printf("[dstat]\n  %x\n  %x\n", it.Key(), it.Value())
+			fmt.Printf("[dstat] K:%x V:%x\n", it.Key(), it.Value())
 
 		case db.KeyTypeFolderStatistic:
-			fmt.Printf("[fstat]\n  %x\n  %x\n", it.Key(), it.Value())
+			fmt.Printf("[fstat] K:%x V:%x\n", it.Key(), it.Value())
 
 		case db.KeyTypeVirtualMtime:
-			fmt.Printf("[mtime]\n  %x\n  %x\n", it.Key(), it.Value())
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			val := it.Value()
+			var real, virt time.Time
+			real.UnmarshalBinary(val[:len(val)/2])
+			virt.UnmarshalBinary(val[len(val)/2:])
+			fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, real, virt)
+
+		case db.KeyTypeFolderIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
+
+		case db.KeyTypeDeviceIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			val := it.Value()
+			if len(val) == 0 {
+				fmt.Printf("[deviceidx] K:%d V:<nil>\n", key)
+			} else {
+				dev := protocol.DeviceIDFromBytes(val)
+				fmt.Printf("[deviceidx] K:%d V:%s\n", key, dev)
+			}
 
 		default:
 			fmt.Printf("[???]\n  %x\n  %x\n", it.Key(), it.Value())

cmd/stindex/dumpsize.go

@@ -2,17 +2,16 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
 	"container/heap"
+	"encoding/binary"
 	"fmt"
 
 	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syndtr/goleveldb/leveldb"
 )
 
 type SizedElement struct {
@@ -38,33 +37,31 @@ func (h *ElementHeap) Pop() interface{} {
 	return x
 }
 
-func dumpsize(ldb *leveldb.DB) {
+func dumpsize(ldb *db.Lowlevel) {
 	h := &ElementHeap{}
 	heap.Init(h)
 
 	it := ldb.NewIterator(nil, nil)
-	var dev protocol.DeviceID
 	var ele SizedElement
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {
 		case db.KeyTypeDevice:
-			folder := nulString(key[1 : 1+64])
-			devBytes := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			copy(dev[:], devBytes)
-			ele.key = fmt.Sprintf("DEVICE:%s:%s:%s", dev, folder, name)
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+			ele.key = fmt.Sprintf("DEVICE:%d:%d:%s", folder, device, name)
 
 		case db.KeyTypeGlobal:
-			folder := nulString(key[1 : 1+64])
-			name := nulString(key[1+64:])
-			ele.key = fmt.Sprintf("GLOBAL:%s:%s", folder, name)
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			ele.key = fmt.Sprintf("GLOBAL:%d:%s", folder, name)
 
 		case db.KeyTypeBlock:
-			folder := nulString(key[1 : 1+64])
-			hash := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			ele.key = fmt.Sprintf("BLOCK:%s:%x:%s", folder, hash, name)
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[1+4 : 1+4+32]
+			name := nulString(key[1+4+32:])
+			ele.key = fmt.Sprintf("BLOCK:%d:%x:%s", folder, hash, name)
 
 		case db.KeyTypeDeviceStatistic:
 			ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])
@@ -75,6 +72,14 @@ func dumpsize(ldb *leveldb.DB) {
 		case db.KeyTypeVirtualMtime:
 			ele.key = fmt.Sprintf("MTIME:%s", key[1:])
 
+		case db.KeyTypeFolderIdx:
+			id := binary.BigEndian.Uint32(key[1:])
+			ele.key = fmt.Sprintf("FOLDERIDX:%d", id)
+
+		case db.KeyTypeDeviceIdx:
+			id := binary.BigEndian.Uint32(key[1:])
+			ele.key = fmt.Sprintf("DEVICEIDX:%d", id)
+
 		default:
 			ele.key = fmt.Sprintf("UNKNOWN:%x", key)
 		}

cmd/stindex/idxck.go

@@ -0,0 +1,242 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+type fileInfoKey struct {
+	folder uint32
+	device uint32
+	name   string
+}
+
+type globalKey struct {
+	folder uint32
+	name   string
+}
+
+type sequenceKey struct {
+	folder   uint32
+	sequence uint64
+}
+
+func idxck(ldb *db.Lowlevel) (success bool) {
+	folders := make(map[uint32]string)
+	devices := make(map[uint32]string)
+	deviceToIDs := make(map[string]uint32)
+	fileInfos := make(map[fileInfoKey]protocol.FileInfo)
+	globals := make(map[globalKey]db.VersionList)
+	sequences := make(map[sequenceKey]string)
+	needs := make(map[globalKey]struct{})
+	var localDeviceKey uint32
+	success = true
+
+	it := ldb.NewIterator(nil, nil)
+	for it.Next() {
+		key := it.Key()
+		switch key[0] {
+		case db.KeyTypeDevice:
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+
+			var f protocol.FileInfo
+			err := f.Unmarshal(it.Value())
+			if err != nil {
+				fmt.Println("Unable to unmarshal FileInfo:", err)
+				success = false
+				continue
+			}
+
+			fileInfos[fileInfoKey{folder, device, name}] = f
+
+		case db.KeyTypeGlobal:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			var flv db.VersionList
+			if err := flv.Unmarshal(it.Value()); err != nil {
+				fmt.Println("Unable to unmarshal VersionList:", err)
+				success = false
+				continue
+			}
+			globals[globalKey{folder, name}] = flv
+
+		case db.KeyTypeFolderIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			folders[key] = string(it.Value())
+
+		case db.KeyTypeDeviceIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			devices[key] = string(it.Value())
+			deviceToIDs[string(it.Value())] = key
+			if bytes.Equal(it.Value(), protocol.LocalDeviceID[:]) {
+				localDeviceKey = key
+			}
+
+		case db.KeyTypeSequence:
+			folder := binary.BigEndian.Uint32(key[1:])
+			seq := binary.BigEndian.Uint64(key[5:])
+			val := it.Value()
+			sequences[sequenceKey{folder, seq}] = string(val[9:])
+
+		case db.KeyTypeNeed:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			needs[globalKey{folder, name}] = struct{}{}
+		}
+	}
+
+	if localDeviceKey == 0 {
+		fmt.Println("Missing key for local device in device index (bailing out)")
+		success = false
+		return
+	}
+
+	for fk, fi := range fileInfos {
+		if fk.name != fi.Name {
+			fmt.Printf("Mismatching FileInfo name, %q (key) != %q (actual)\n", fk.name, fi.Name)
+			success = false
+		}
+
+		folder := folders[fk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for FileInfo %q\n", fk.folder, fk.name)
+			success = false
+			continue
+		}
+		if devices[fk.device] == "" {
+			fmt.Printf("Unknown device ID %d for FileInfo %q, folder %q\n", fk.folder, fk.name, folder)
+			success = false
+		}
+
+		if fk.device == localDeviceKey {
+			name, ok := sequences[sequenceKey{fk.folder, uint64(fi.Sequence)}]
+			if !ok {
+				fmt.Printf("Sequence entry missing for FileInfo %q, folder %q, seq %d\n", fi.Name, folder, fi.Sequence)
+				success = false
+				continue
+			}
+			if name != fi.Name {
+				fmt.Printf("Sequence entry refers to wrong name, %q (seq) != %q (FileInfo), folder %q, seq %d\n", name, fi.Name, folder, fi.Sequence)
+				success = false
+			}
+		}
+	}
+
+	for gk, vl := range globals {
+		folder := folders[gk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
+			success = false
+		}
+		for i, fv := range vl.Versions {
+			dev, ok := deviceToIDs[string(fv.Device)]
+			if !ok {
+				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, fv.Device)
+				success = false
+			}
+			fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
+			if !ok {
+				fmt.Printf("VersionList %q, folder %q, entry %d refers to unknown FileInfo\n", gk.name, folder, i)
+				success = false
+			}
+			if !fi.Version.Equal(fv.Version) {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Version, fi.Version)
+				success = false
+			}
+			if fi.IsInvalid() != fv.Invalid {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Invalid, fi.IsInvalid())
+				success = false
+			}
+		}
+
+		// If we need this file we should have a need entry for it. False
+		// positives from needsLocally for deleted files, where we might
+		// legitimately lack an entry if we never had it, and ignored files.
+		if needsLocally(vl) {
+			_, ok := needs[gk]
+			if !ok {
+				dev := deviceToIDs[string(vl.Versions[0].Device)]
+				fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
+				if !fi.IsDeleted() && !fi.IsIgnored() {
+					fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
+				}
+			}
+		}
+	}
+
+	seenSeq := make(map[fileInfoKey]uint64)
+	for sk, name := range sequences {
+		folder := folders[sk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for sequence entry %d, %q\n", sk.folder, sk.sequence, name)
+			success = false
+			continue
+		}
+
+		if prev, ok := seenSeq[fileInfoKey{folder: sk.folder, name: name}]; ok {
+			fmt.Printf("Duplicate sequence entry for %q, folder %q, seq %d (prev %d)\n", name, folder, sk.sequence, prev)
+			success = false
+		}
+		seenSeq[fileInfoKey{folder: sk.folder, name: name}] = sk.sequence
+
+		fi, ok := fileInfos[fileInfoKey{sk.folder, localDeviceKey, name}]
+		if !ok {
+			fmt.Printf("Missing FileInfo for sequence entry %d, folder %q, %q\n", sk.sequence, folder, name)
+			success = false
+			continue
+		}
+		if fi.Sequence != int64(sk.sequence) {
+			fmt.Printf("Sequence mismatch for %q, folder %q, %d (key) != %d (FileInfo)\n", name, folder, sk.sequence, fi.Sequence)
+			success = false
+		}
+	}
+
+	for nk := range needs {
+		folder := folders[nk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for need entry %q\n", nk.folder, nk.name)
+			success = false
+			continue
+		}
+
+		vl, ok := globals[nk]
+		if !ok {
+			fmt.Printf("Missing global for need entry %q, folder %q\n", nk.name, folder)
+			success = false
+			continue
+		}
+
+		if !needsLocally(vl) {
+			fmt.Printf("Need entry for file we don't need, %q, folder %q\n", nk.name, folder)
+			success = false
+		}
+	}
+
+	return
+}
+
+func needsLocally(vl db.VersionList) bool {
+	var lv *protocol.Vector
+	for _, fv := range vl.Versions {
+		if bytes.Equal(fv.Device, protocol.LocalDeviceID[:]) {
+			lv = &fv.Version
+			break
+		}
+	}
+	if lv == nil {
+		return true // proviosinally, it looks like we need the file
+	}
+	return !lv.GreaterEqual(vl.Versions[0].Version)
+}

cmd/stindex/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -13,8 +13,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syncthing/syncthing/lib/db"
 )
 
 func main() {
@@ -22,22 +21,16 @@ func main() {
 	log.SetFlags(0)
 	log.SetOutput(os.Stdout)
 
-	flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize")
+	flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize, idxck")
 
 	flag.Parse()
 
 	path := flag.Arg(0)
 	if path == "" {
-		path = filepath.Join(defaultConfigDir(), "index-v0.11.0.db")
+		path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
 	}
 
-	fmt.Println("Path:", path)
-
-	ldb, err := leveldb.OpenFile(path, &opt.Options{
-		ErrorIfMissing:         true,
-		Strict:                 opt.StrictAll,
-		OpenFilesCacheCapacity: 100,
-	})
+	ldb, err := db.OpenRO(path)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -46,6 +39,10 @@ func main() {
 		dump(ldb)
 	} else if mode == "dumpsize" {
 		dumpsize(ldb)
+	} else if mode == "idxck" {
+		if !idxck(ldb) {
+			os.Exit(1)
+		}
 	} else {
 		fmt.Println("Unknown mode")
 	}

cmd/stindex/util.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -12,7 +12,7 @@ import (
 	"path/filepath"
 	"runtime"
 
-	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/fs"
 )
 
 func nulString(bs []byte) string {
@@ -33,7 +33,7 @@ func defaultConfigDir() string {
 		return filepath.Join(os.Getenv("AppData"), "Syncthing")
 
 	case "darwin":
-		dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
+		dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -43,7 +43,7 @@ func defaultConfigDir() string {
 		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
 			return filepath.Join(xdgCfg, "syncthing")
 		}
-		dir, err := osutil.ExpandTilde("~/.config/syncthing")
+		dir, err := fs.ExpandTilde("~/.config/syncthing")
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/strelaypoolsrv/LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015 Tristan Rice
+Copyright (c) 2015 The Syncthing Project
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

cmd/strelaypoolsrv/README.md

@@ -0,0 +1,24 @@
+# relaypoolsrv
+
+This is the relay pool server for the `syncthing` project, which allows
+community hosted [relaysrv](https://github.com/syncthing/relaysrv)'s to join
+the public pool.
+
+Servers that join the pool are then advertised to users of `syncthing` as
+potential connection points for those who are unable to connect directly due
+to NAT or firewall issues.
+
+There is very little reason why you'd want to run this yourself, as
+`relaypoolsrv` is just used for announcement and lookup of public relay
+servers. If you are looking to setup a private or a public relay, please
+check the documentation for
+[relaysrv](https://github.com/syncthing/relaysrv), which also explains how
+to join the default public pool.
+
+See `relaypoolsrv -help` for configuration options.
+
+##### Third-party attributions
+
+[oschwald/geoip2-golang](https://github.com/oschwald/geoip2-golang), [oschwald/maxminddb-golang](https://github.com/oschwald/maxminddb-golang), Copyright (C) 2015 [Gregory J. Oschwald](mailto:oschwald@gmail.com).
+
+[lib/pq](https://github.com/lib/pq)</a>, Copyright (C) 2011-2013 'pq' Contributors Portions Copyright (C) 2011 Blake Mizerany.

cmd/strelaypoolsrv/auto/.gitignore

@@ -0,0 +1 @@
+gui.files.go

cmd/strelaypoolsrv/auto/doc.go

@@ -0,0 +1,10 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//go:generate go run ../../../script/genassets.go -o gui.files.go ../gui
+
+// Package auto contains auto generated files for web assets.
+package auto

cmd/strelaypoolsrv/gui/index.html

@@ -0,0 +1,386 @@
+<!DOCTYPE html>
+
+<html lang="en" ng-app="syncthing" ng-controller="relayDataController">
+  <head>
+    <meta charset="utf-8"/>
+    <meta http-equiv="X-UA-Compatible" content="IE=edge"/>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+    <meta name="description" content=""/>
+    <meta name="author" content=""/>
+
+    <title>Relay stats</title>
+    <link href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
+    <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.0.13/css/all.css"/>
+
+    <style>
+      #map {
+        height: 600px;
+      }
+      .ng-cloak {
+        display: none;
+      }
+      table {
+        font-size: 11px !important;
+        width: 100%;
+        border: 1px;
+
+      }
+      td {
+        padding: 0px !important;
+      }
+      tfoot td {
+        font-weight: bold;
+      }
+    </style>
+  </head>
+
+  <body class="ng-cloak">
+    <div class="container">
+      <h1>Relay Pool Data</h1>
+      <div ng-if="relays === undefined" class="text-center">
+        <img src="//cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
+        <p>Please wait while we gather data</p>
+      </div>
+      <div>
+        <div ng-show="relays !== undefined" class="ng-hide">
+          <p>
+            Currently {{ relays.length }} relays online ({{ totals.goMaxProcs }} cores in total).
+          </p>
+        </div>
+        <div id="map"></div> <!-- Can't hide the map, otherwise it freaks out -->
+        <p>The circle size represents how much bytes the relay transferred relative to other relays</p>
+      </div>
+      <div>
+        <table class="table table-striped table-condensed table">
+          <thead>
+            <tr>
+              <th rowspan="2">Address</td>
+              <th rowspan="2">
+                <a ng-click="sortType = 'stats.numActiveSessions'; sortReverse = !sortReverse">
+                  Sessions
+                  <span ng-show="sortType == 'stats.numActiveSessions' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.numActiveSessions' && sortReverse" class="fas fa-caret-up"></span>
+                </a>
+              </th>
+              <th rowspan="2">
+                <a ng-click="sortType = 'stats.numConnections'; sortReverse = !sortReverse">
+                  Connections
+                  <span ng-show="sortType == 'stats.numConnections' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.numConnections' && sortReverse" class="fas fa-caret-up"></span>
+                </a>
+              </th>
+              <th rowspan="2">
+                <a ng-click="sortType = 'stats.bytesProxied'; sortReverse = !sortReverse">
+                  Data relayed
+                  <span ng-show="sortType == 'stats.bytesProxied' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.bytesProxied' && sortReverse" class="fas fa-caret-up"></span>
+                </a>
+              </th>
+              <th colspan="6" class="text-center">Transfer rate in the last period</th>
+              <th rowspan="2">
+                <a ng-click="sortType = 'stats.uptimeSeconds'; sortReverse = !sortReverse">
+                  Uptime hours
+                  <span ng-show="sortType == 'stats.uptimeSeconds' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.uptimeSeconds' && sortReverse" class="fas fa-caret-up"></span>
+                </a>
+              </th>
+              <th rowspan="2">
+                <a ng-click="sortType = 'stats.options[\'provided-by\'] || \'\''; sortReverse = !sortReverse">
+                  Provided by
+                  <span ng-show="sortType == 'stats.options[\'provided-by\'] || \'\'' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.options[\'provided-by\'] || \'\'' && sortReverse" class="fas fa-caret-up"></span>
+                </a>
+              </th>
+            </tr>
+            <tr>
+              <th>
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[0]'; sortReverse = !sortReverse">
+                  10s
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[0]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[0]' && sortReverse" class="fas fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[1]'; sortReverse = !sortReverse">
+                  1m
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[1]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[1]' && sortReverse" class="fas fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[2]'; sortReverse = !sortReverse">
+                  5m
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[2]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[2]' && sortReverse" class="fas fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[3]'; sortReverse = !sortReverse">
+                  15m
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[3]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[3]' && sortReverse" class="fas fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[4]'; sortReverse = !sortReverse">
+                  30m
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[4]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[4]' && sortReverse" class="fas fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[5]'; sortReverse = !sortReverse">
+                  60m
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[5]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[5]' && sortReverse" class="fas fa-caret-up"></span>
+                </a>
+              </th>
+            </tr>
+          </thead>
+          <tbody>
+            <tr ng-repeat="relay in relays | orderBy:sortType:sortReverse:sortCompare" ng-mouseover="relay.showMarker()" ng-mouseleave="relay.hideMarker()">
+              <td>{{ relay.address }}</td>
+              <td ng-if="!relay.stats" colspan="11"></td>
+              <td ng-if-start="relay.stats">{{ relay.stats.numActiveSessions }}</td>
+              <td>{{ relay.stats.numConnections }}</td>
+              <td>{{ relay.stats.bytesProxied | bytes }}</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
+              <td ng-if="relay.stats.uptimeSeconds != undefined">{{ relay.stats.uptimeSeconds/60/60 | number:0 }}</td>
+              <td ng-if="relay.stats.uptimeSeconds == undefined"></td>
+              <td title="{{ relay.stats.options['provided-by'] || '' }}" ng-if-end>
+                {{ relay.stats.options['provided-by'] || '' | limitTo:50 }}
+                <span ng-if="(relay.stats.options['provided-by'] || '').length > 50">&hellip;
+              </td>
+            </tr>
+          </tbody>
+          <tfoot>
+            <tr>
+              <td>Totals</td>
+              <td>{{ totals.numActiveSessions }}</td>
+              <td>{{ totals.numConnections }}</td>
+              <td>{{ totals.bytesProxied | bytes }}</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
+              <td>{{ totals.uptimeSeconds/60/60 | number:0 }} hours</td>
+              <td>{{ relays.length }} relays</td>
+            </tr>
+          </tfoor>
+        </table>
+      </div>
+      <hr>
+      <p>
+        This product includes GeoLite2 data created by MaxMind, available from
+        <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
+      </p>
+    </div>
+
+
+    <script type="text/javascript" src="//code.jquery.com/jquery-2.1.4.min.js"></script>
+    <script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
+    <script type="text/javascript" src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
+    <script type="text/javascript" src="//maps.googleapis.com/maps/api/js?key=AIzaSyDk5WJ8s7ueLKb99X5DbQ-vkWtPDAKqYs0"></script>
+  </body>
+
+  <script>
+  angular.module('syncthing', [
+  ])
+  .config(['$httpProvider', function($httpProvider) {
+    $httpProvider.defaults.timeout = 5000;
+  }])
+  .filter('bytes', function() {
+    return function(bytes, precision) {
+      if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) return '-';
+      if (typeof precision === 'undefined') precision = 1;
+
+      var units = ['bytes', 'kB', 'MB', 'GB', 'TB', 'PB'],
+          number = Math.floor(Math.log(bytes) / Math.log(1024));
+
+      var value = (bytes / Math.pow(1000, Math.floor(number)));
+      if (!isFinite(value)) {
+        value = 0;
+        precision = 0;
+      }
+      if (!isFinite(number)) {
+        units = 'bytes';
+      } else {
+        units = units[number];
+      }
+      return value.toFixed(precision) + ' ' + units;
+    }
+  })
+  .controller('relayDataController', ['$scope', '$rootScope', '$http', '$q', '$compile', '$timeout', function($scope, $rootScope, $http, $q, $compile, $timeout) {
+    $scope.totals = {
+      bytesProxied: 0,
+      goMaxProcs: 0,
+      kbps10s1m5m15m30m60m: [0, 0, 0, 0, 0, 0],
+      numActiveSessions: 0,
+      numConnections: 0,
+      numPendingSessionKeys: 0,
+      numProxies: 0,
+      uptimeSeconds: 0,
+    };
+    $scope.map = new google.maps.Map(document.getElementById('map'), {
+      zoom: 1,
+      mapTypeId: google.maps.MapTypeId.ROADMAP
+    });
+    $scope.mapBounds = new google.maps.LatLngBounds();
+    $scope.tooltipTemplate = $('#infoTemplate').html();
+    $scope.usedLocations = {};
+    $scope.sortType = 'stats.numActiveSessions';
+    $scope.sortReverse = true;
+    $scope.sortCompare = function(a, b) {
+      if (a.value == b.value) {
+        return 0;
+      }
+      if (a.type == "undefined" || a.type == "null") {
+        return -1;
+      }
+      if (b.type == "undefined" || b.type == "null") {
+        return 1;
+      }
+      return a.value > b.value ? 1 : -1;
+    }
+
+    $http.get("/endpoint").then(function(response) {
+      $scope.relays = response.data.relays;
+
+      angular.forEach($scope.relays, function(relay) {
+        relay.uri = constructURI(relay.url);
+        relay.address = relay.url.split('/')[2];
+
+        addMarkerToMap(relay);
+
+        if (relay.stats) {
+          angular.forEach($scope.totals, function(value, key) {
+            if (typeof $scope.totals[key] == 'number') {
+              $scope.totals[key] += relay.stats[key];
+            } else if (typeof $scope.totals[key] == 'object' && $scope.totals[key] instanceof Array) {
+              angular.forEach($scope.totals[key], function(value, index) {
+                $scope.totals[key][index] += relay.stats[key][index];
+              });
+            }
+          });
+        }
+      });
+
+      // After the totals were calculated, add circles.
+      angular.forEach($scope.relays, function(relay) {
+        if (relay.stats) {
+          addCircleToMap(relay);
+        }
+      });
+
+      $scope.map.fitBounds($scope.mapBounds);
+      if ($scope.relays.length == 1) {
+        $scope.map.setZoom(13);
+      }
+    });
+
+    function addMarkerToMap(relay) {
+        var loc = relay.location.latitude + "," + relay.location.longitude;
+
+        // Deal with overlapping markers
+        while (loc in $scope.usedLocations) {
+          var locParts = loc.split(',');
+          locParts = [parseFloat(locParts[0]), parseFloat(locParts[1])];
+          locParts[Math.round(Math.random())] += 0.5 * (Math.random() >= 0.5 ? 1 : -1);
+          loc = locParts.join(',');
+        }
+
+        $scope.usedLocations[loc] = true;
+
+        var locParts = loc.split(',');
+
+        relay.marker = new google.maps.Marker({
+          map: $scope.map,
+          position: new google.maps.LatLng(locParts[0], locParts[1]),
+          title: relay.url,
+        });
+
+        var scope = $rootScope.$new(true);
+        scope.relay = relay;
+
+        relay.marker.info = new google.maps.InfoWindow({
+          content: $compile($scope.tooltipTemplate)(scope)[0],
+        });
+
+        relay.showMarker = function() {
+          relay.marker.info.open($scope.map, relay.marker);
+        }
+
+        relay.hideMarker = function() {
+          relay.marker.info.close();
+        }
+
+        relay.marker.addListener('mouseover', relay.showMarker);
+        relay.marker.addListener('mouseout', relay.hideMarker);
+
+        $scope.mapBounds.extend(relay.marker.position);
+    }
+
+    function addCircleToMap(relay) {
+      relay.marker.circle = new google.maps.Circle({
+        strokeColor: '#FF0000',
+        strokeOpacity: 0.8,
+        strokeWeight: 2,
+        fillColor: '#FF0000',
+        fillOpacity: 0.35,
+        map: $scope.map,
+        center: relay.marker.position,
+        radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000
+      });
+    }
+
+    function constructURI(url) {
+      var uri = document.createElement('a');
+
+      // HAX, otherwise doesn't work
+      uri.href = url.replace('relay://', 'http://');
+
+      // Convert query string to object
+      uri.args = {};
+      angular.forEach(uri.search.replace(/^\?/, '').split('&'), function(query) {
+          var split = query.split('=');
+          uri.args[split[0]] = split[1];
+      });
+
+      return uri;
+    }
+  }]);
+  </script>
+
+  <script type="text/template" id="infoTemplate">
+    <div>
+      <p><b>{{ relay.uri.hostname }}</b> <span ng-if="relay.stats.options['provided-by']">provided by <u>{{ relay.stats.options['provided-by'] }}</u></span></p>
+      <div ng-if="relay.stats">
+        <span ng-if="relay.stats.startTime">Start time: {{ relay.stats.startTime | date:"medium" }}</br></span>
+        <span ng-if="relay.stats.bytesProxied != undefined">Proxied: {{ relay.stats.bytesProxied | bytes }}</br></span>
+        <span ng-if="relay.stats.numActiveSessions != undefined">Sessions: {{ relay.stats.numActiveSessions }}</br></span>
+        <span ng-if="relay.stats.numConnections != undefined">Clients: {{ relay.stats.numConnections }}</br></span>
+        <span ng-if="relay.stats.options.pools">Pools: {{ relay.stats.options.pools.join(', ') }}</br></span>
+        <span ng-if="relay.stats.options['global-rate'] != undefined">
+          <span ng-if="relay.stats.options['global-rate'] > 0">Global rate limit: {{ relay.stats.options['global-rate'] | bytes }}/s</span>
+          <span ng-if="relay.stats.options['global-rate'] == 0">Global rate limit: unlimited</span>
+          <br/>
+        </span>
+        <span ng-if="relay.stats.options['per-session-rate'] != undefined">
+          <span ng-if="relay.stats.options['per-session-rate'] > 0">Session rate limit: {{ relay.stats.options['per-session-rate'] | bytes }}/s</span>
+          <span ng-if="relay.stats.options['per-session-rate'] == 0">Session rate limit: unlimited</span>
+          <br/>
+        </span>
+      </div>
+      <div ng-if="!relay.stats">
+        Data unavailable.
+      <div>
+    </div>
+  </script>
+</html>

cmd/strelaypoolsrv/main.go

@@ -0,0 +1,684 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+//go:generate go run ../../script/genassets.go gui >auto/gui.go
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"crypto/tls"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"mime"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/golang/groupcache/lru"
+	"github.com/oschwald/geoip2-golang"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
+	"github.com/syncthing/syncthing/lib/rand"
+	"github.com/syncthing/syncthing/lib/relay/client"
+	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/tlsutil"
+	"golang.org/x/time/rate"
+)
+
+type location struct {
+	Latitude  float64 `json:"latitude"`
+	Longitude float64 `json:"longitude"`
+	City      string  `json:"city"`
+	Country   string  `json:"country"`
+	Continent string  `json:"continent"`
+}
+
+type relay struct {
+	URL            string   `json:"url"`
+	Location       location `json:"location"`
+	uri            *url.URL
+	Stats          *stats    `json:"stats"`
+	StatsRetrieved time.Time `json:"statsRetrieved"`
+}
+
+type stats struct {
+	StartTime          time.Time `json:"startTime"`
+	UptimeSeconds      int       `json:"uptimeSeconds"`
+	PendingSessionKeys int       `json:"numPendingSessionKeys"`
+	ActiveSessions     int       `json:"numActiveSessions"`
+	Connections        int       `json:"numConnections"`
+	Proxies            int       `json:"numProxies"`
+	BytesProxied       int       `json:"bytesProxied"`
+	GoVersion          string    `json:"goVersion"`
+	GoOS               string    `json:"goOS"`
+	GoArch             string    `json:"goArch"`
+	GoMaxProcs         int       `json:"goMaxProcs"`
+	GoRoutines         int       `json:"goNumRoutine"`
+	Rates              []int64   `json:"kbps10s1m5m15m30m60m"`
+	Options            struct {
+		NetworkTimeout int      `json:"network-timeout"`
+		PintInterval   int      `json:"ping-interval"`
+		MessageTimeout int      `json:"message-timeout"`
+		SessionRate    int      `json:"per-session-rate"`
+		GlobalRate     int      `json:"global-rate"`
+		Pools          []string `json:"pools"`
+		ProvidedBy     string   `json:"provided-by"`
+	} `json:"options"`
+}
+
+func (r relay) String() string {
+	return r.URL
+}
+
+type request struct {
+	relay      *relay
+	result     chan result
+	queueTimer *prometheus.Timer
+}
+
+type result struct {
+	err      error
+	eviction time.Duration
+}
+
+var (
+	testCert        tls.Certificate
+	knownRelaysFile = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
+	listen          = ":80"
+	dir             string
+	evictionTime    = time.Hour
+	debug           bool
+	getLRUSize      = 10 << 10
+	getLimitBurst   = 10
+	getLimitAvg     = 2
+	postLRUSize     = 1 << 10
+	postLimitBurst  = 2
+	postLimitAvg    = 2
+	getLimit        time.Duration
+	postLimit       time.Duration
+	permRelaysFile  string
+	ipHeader        string
+	geoipPath       string
+	proto           string
+	statsRefresh    = time.Minute / 2
+
+	getMut      = sync.NewRWMutex()
+	getLRUCache *lru.Cache
+
+	postMut      = sync.NewRWMutex()
+	postLRUCache *lru.Cache
+
+	requests = make(chan request, 10)
+
+	mut             = sync.NewRWMutex()
+	knownRelays     = make([]*relay, 0)
+	permanentRelays = make([]*relay, 0)
+	evictionTimers  = make(map[string]*time.Timer)
+)
+
+const (
+	httpStatusEnhanceYourCalm = 429
+)
+
+func main() {
+	flag.StringVar(&listen, "listen", listen, "Listen address")
+	flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
+	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
+	flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
+	flag.IntVar(&getLRUSize, "get-limit-cache", getLRUSize, "Get request limiter cache size")
+	flag.IntVar(&getLimitAvg, "get-limit-avg", getLimitAvg, "Allowed average get request rate, per 10 s")
+	flag.IntVar(&getLimitBurst, "get-limit-burst", getLimitBurst, "Allowed burst get requests")
+	flag.IntVar(&postLRUSize, "post-limit-cache", postLRUSize, "Post request limiter cache size")
+	flag.IntVar(&postLimitAvg, "post-limit-avg", postLimitAvg, "Allowed average post request rate, per minute")
+	flag.IntVar(&postLimitBurst, "post-limit-burst", postLimitBurst, "Allowed burst post requests")
+	flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
+	flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
+	flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
+	flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
+	flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
+
+	flag.Parse()
+
+	getLimit = 10 * time.Second / time.Duration(getLimitAvg)
+	postLimit = time.Minute / time.Duration(postLimitAvg)
+
+	getLRUCache = lru.New(getLRUSize)
+	postLRUCache = lru.New(postLRUSize)
+
+	var listener net.Listener
+	var err error
+
+	if permRelaysFile != "" {
+		permanentRelays = loadRelays(permRelaysFile)
+	}
+
+	testCert = createTestCertificate()
+
+	go requestProcessor()
+
+	// Load relays from cache in the background.
+	// Load them in a serial fashion to make sure any genuine requests
+	// are not dropped.
+	go func() {
+		for _, relay := range loadRelays(knownRelaysFile) {
+			resultChan := make(chan result)
+			requests <- request{relay, resultChan, nil}
+			result := <-resultChan
+			if result.err != nil {
+				relayTestsTotal.WithLabelValues("failed").Inc()
+			} else {
+				relayTestsTotal.WithLabelValues("success").Inc()
+			}
+		}
+		// Run the the stats refresher once the relays are loaded.
+		statsRefresher(statsRefresh)
+	}()
+
+	if dir != "" {
+		if debug {
+			log.Println("Starting TLS listener on", listen)
+		}
+		certFile, keyFile := filepath.Join(dir, "http-cert.pem"), filepath.Join(dir, "http-key.pem")
+		var cert tls.Certificate
+		cert, err = tls.LoadX509KeyPair(certFile, keyFile)
+		if err != nil {
+			log.Fatalln("Failed to load HTTP X509 key pair:", err)
+		}
+
+		tlsCfg := &tls.Config{
+			Certificates: []tls.Certificate{cert},
+			MinVersion:   tls.VersionTLS10, // No SSLv3
+			CipherSuites: []uint16{
+				// No RC4
+				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
+				tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,
+			},
+		}
+
+		listener, err = tls.Listen(proto, listen, tlsCfg)
+	} else {
+		if debug {
+			log.Println("Starting plain listener on", listen)
+		}
+		listener, err = net.Listen(proto, listen)
+	}
+
+	if err != nil {
+		log.Fatalln("listen:", err)
+	}
+
+	handler := http.NewServeMux()
+	handler.HandleFunc("/", handleAssets)
+	handler.HandleFunc("/endpoint", handleRequest)
+	handler.HandleFunc("/metrics", handleMetrics)
+
+	srv := http.Server{
+		Handler:     handler,
+		ReadTimeout: 10 * time.Second,
+	}
+
+	err = srv.Serve(listener)
+	if err != nil {
+		log.Fatalln("serve:", err)
+	}
+}
+
+func handleMetrics(w http.ResponseWriter, r *http.Request) {
+	timer := prometheus.NewTimer(metricsRequestsSeconds)
+	// Acquire the mutex just to make sure we're not caught mid-way stats collection
+	mut.RLock()
+	promhttp.Handler().ServeHTTP(w, r)
+	mut.RUnlock()
+	timer.ObserveDuration()
+}
+
+func handleAssets(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Cache-Control", "no-cache, must-revalidate")
+
+	assets := auto.Assets()
+	path := r.URL.Path[1:]
+	if path == "" {
+		path = "index.html"
+	}
+
+	bs, ok := assets[path]
+	if !ok {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
+	etag := fmt.Sprintf("%d", auto.Generated)
+	modified := time.Unix(auto.Generated, 0).UTC()
+
+	w.Header().Set("Last-Modified", modified.Format(http.TimeFormat))
+	w.Header().Set("Etag", etag)
+
+	mtype := mimeTypeForFile(path)
+	if len(mtype) != 0 {
+		w.Header().Set("Content-Type", mtype)
+	}
+
+	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modified.Add(time.Second).After(t) {
+		w.WriteHeader(http.StatusNotModified)
+		return
+	}
+
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+		w.Header().Set("Content-Encoding", "gzip")
+	} else {
+		// ungzip if browser not send gzip accepted header
+		var gr *gzip.Reader
+		gr, _ = gzip.NewReader(bytes.NewReader(bs))
+		bs, _ = ioutil.ReadAll(gr)
+		gr.Close()
+	}
+	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs)))
+
+	w.Write(bs)
+}
+
+func mimeTypeForFile(file string) string {
+	// We use a built in table of the common types since the system
+	// TypeByExtension might be unreliable. But if we don't know, we delegate
+	// to the system.
+	ext := filepath.Ext(file)
+	switch ext {
+	case ".htm", ".html":
+		return "text/html"
+	case ".css":
+		return "text/css"
+	case ".js":
+		return "application/javascript"
+	case ".json":
+		return "application/json"
+	case ".png":
+		return "image/png"
+	case ".ttf":
+		return "application/x-font-ttf"
+	case ".woff":
+		return "application/x-font-woff"
+	case ".svg":
+		return "image/svg+xml"
+	default:
+		return mime.TypeByExtension(ext)
+	}
+}
+
+func handleRequest(w http.ResponseWriter, r *http.Request) {
+	timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
+
+	lw := NewLoggingResponseWriter(w)
+
+	defer func() {
+		timer.ObserveDuration()
+		apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
+	}()
+
+	if ipHeader != "" {
+		r.RemoteAddr = r.Header.Get(ipHeader)
+	}
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	switch r.Method {
+	case "GET":
+		if limit(r.RemoteAddr, getLRUCache, getMut, getLimit, getLimitBurst) {
+			w.WriteHeader(httpStatusEnhanceYourCalm)
+			return
+		}
+		handleGetRequest(w, r)
+	case "POST":
+		if limit(r.RemoteAddr, postLRUCache, postMut, postLimit, postLimitBurst) {
+			w.WriteHeader(httpStatusEnhanceYourCalm)
+			return
+		}
+		handlePostRequest(w, r)
+	default:
+		if debug {
+			log.Println("Unhandled HTTP method", r.Method)
+		}
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func handleGetRequest(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	mut.RLock()
+	relays := append(permanentRelays, knownRelays...)
+	mut.RUnlock()
+
+	// Shuffle
+	rand.Shuffle(relays)
+
+	json.NewEncoder(w).Encode(map[string][]*relay{
+		"relays": relays,
+	})
+}
+
+func handlePostRequest(w http.ResponseWriter, r *http.Request) {
+	var newRelay relay
+	err := json.NewDecoder(r.Body).Decode(&newRelay)
+	r.Body.Close()
+
+	if err != nil {
+		if debug {
+			log.Println("Failed to parse payload")
+		}
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	uri, err := url.Parse(newRelay.URL)
+	if err != nil {
+		if debug {
+			log.Println("Failed to parse URI", newRelay.URL)
+		}
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	host, port, err := net.SplitHostPort(uri.Host)
+	if err != nil {
+		if debug {
+			log.Println("Failed to split URI", newRelay.URL)
+		}
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	// Get the IP address of the client
+	rhost := r.RemoteAddr
+	if host, _, err := net.SplitHostPort(rhost); err == nil {
+		rhost = host
+	}
+
+	ip := net.ParseIP(host)
+	// The client did not provide an IP address, use the IP address of the client.
+	if ip == nil || ip.IsUnspecified() {
+		uri.Host = net.JoinHostPort(rhost, port)
+		newRelay.URL = uri.String()
+	} else if host != rhost {
+		if debug {
+			log.Println("IP address advertised does not match client IP address", r.RemoteAddr, uri)
+		}
+		http.Error(w, fmt.Sprintf("IP advertised %s does not match client IP %s", host, rhost), http.StatusUnauthorized)
+		return
+	}
+
+	newRelay.uri = uri
+
+	for _, current := range permanentRelays {
+		if current.uri.Host == newRelay.uri.Host {
+			if debug {
+				log.Println("Asked to add a relay", newRelay, "which exists in permanent list")
+			}
+			http.Error(w, "Invalid request", http.StatusBadRequest)
+			return
+		}
+	}
+
+	reschan := make(chan result)
+
+	select {
+	case requests <- request{&newRelay, reschan, prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("queue"))}:
+		result := <-reschan
+		if result.err != nil {
+			relayTestsTotal.WithLabelValues("failed").Inc()
+			http.Error(w, result.err.Error(), http.StatusBadRequest)
+			return
+		}
+		relayTestsTotal.WithLabelValues("success").Inc()
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		json.NewEncoder(w).Encode(map[string]time.Duration{
+			"evictionIn": result.eviction,
+		})
+
+	default:
+		relayTestsTotal.WithLabelValues("dropped").Inc()
+		if debug {
+			log.Println("Dropping request")
+		}
+		w.WriteHeader(httpStatusEnhanceYourCalm)
+	}
+}
+
+func requestProcessor() {
+	for request := range requests {
+		if request.queueTimer != nil {
+			request.queueTimer.ObserveDuration()
+		}
+
+		timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test"))
+		handleRelayTest(request)
+		timer.ObserveDuration()
+	}
+}
+
+func handleRelayTest(request request) {
+	if debug {
+		log.Println("Request for", request.relay)
+	}
+	if !client.TestRelay(request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
+		if debug {
+			log.Println("Test for relay", request.relay, "failed")
+		}
+		request.result <- result{fmt.Errorf("connection test failed"), 0}
+		return
+	}
+
+	stats := fetchStats(request.relay)
+	location := getLocation(request.relay.uri.Host)
+
+	mut.Lock()
+	if stats != nil {
+		updateMetrics(request.relay.uri.Host, *stats, location)
+	}
+	request.relay.Stats = stats
+	request.relay.StatsRetrieved = time.Now()
+	request.relay.Location = location
+
+	timer, ok := evictionTimers[request.relay.uri.Host]
+	if ok {
+		if debug {
+			log.Println("Stopping existing timer for", request.relay)
+		}
+		timer.Stop()
+	}
+
+	for i, current := range knownRelays {
+		if current.uri.Host == request.relay.uri.Host {
+			if debug {
+				log.Println("Relay", request.relay, "already exists")
+			}
+
+			// Evict the old entry anyway, as configuration might have changed.
+			last := len(knownRelays) - 1
+			knownRelays[i] = knownRelays[last]
+			knownRelays = knownRelays[:last]
+
+			goto found
+		}
+	}
+
+	if debug {
+		log.Println("Adding new relay", request.relay)
+	}
+
+found:
+
+	knownRelays = append(knownRelays, request.relay)
+	evictionTimers[request.relay.uri.Host] = time.AfterFunc(evictionTime, evict(request.relay))
+
+	mut.Unlock()
+
+	if err := saveRelays(knownRelaysFile, knownRelays); err != nil {
+		log.Println("Failed to write known relays: " + err.Error())
+	}
+
+	request.result <- result{nil, evictionTime}
+}
+
+func evict(relay *relay) func() {
+	return func() {
+		mut.Lock()
+		defer mut.Unlock()
+		if debug {
+			log.Println("Evicting", relay)
+		}
+		for i, current := range knownRelays {
+			if current.uri.Host == relay.uri.Host {
+				if debug {
+					log.Println("Evicted", relay)
+				}
+				last := len(knownRelays) - 1
+				knownRelays[i] = knownRelays[last]
+				knownRelays = knownRelays[:last]
+				deleteMetrics(current.uri.Host)
+			}
+		}
+		delete(evictionTimers, relay.uri.Host)
+	}
+}
+
+func limit(addr string, cache *lru.Cache, lock sync.RWMutex, intv time.Duration, burst int) bool {
+	if host, _, err := net.SplitHostPort(addr); err == nil {
+		addr = host
+	}
+
+	lock.RLock()
+	bkt, ok := cache.Get(addr)
+	lock.RUnlock()
+	if ok {
+		bkt := bkt.(*rate.Limiter)
+		if !bkt.Allow() {
+			// Rate limit
+			return true
+		}
+	} else {
+		lock.Lock()
+		cache.Add(addr, rate.NewLimiter(rate.Every(intv), burst))
+		lock.Unlock()
+	}
+	return false
+}
+
+func loadRelays(file string) []*relay {
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		log.Println("Failed to load relays: " + err.Error())
+		return nil
+	}
+
+	var relays []*relay
+	for _, line := range strings.Split(string(content), "\n") {
+		if len(line) == 0 {
+			continue
+		}
+
+		uri, err := url.Parse(line)
+		if err != nil {
+			if debug {
+				log.Println("Skipping relay", line, "due to parse error", err)
+			}
+			continue
+
+		}
+
+		relays = append(relays, &relay{
+			URL:      line,
+			Location: getLocation(uri.Host),
+			uri:      uri,
+		})
+		if debug {
+			log.Println("Adding relay", line)
+		}
+	}
+	return relays
+}
+
+func saveRelays(file string, relays []*relay) error {
+	var content string
+	for _, relay := range relays {
+		content += relay.uri.String() + "\n"
+	}
+	return ioutil.WriteFile(file, []byte(content), 0777)
+}
+
+func createTestCertificate() tls.Certificate {
+	tmpDir, err := ioutil.TempDir("", "relaypoolsrv")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
+	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv")
+	if err != nil {
+		log.Fatalln("Failed to create test X509 key pair:", err)
+	}
+
+	return cert
+}
+
+func getLocation(host string) location {
+	timer := prometheus.NewTimer(locationLookupSeconds)
+	defer timer.ObserveDuration()
+	db, err := geoip2.Open(geoipPath)
+	if err != nil {
+		return location{}
+	}
+	defer db.Close()
+
+	addr, err := net.ResolveTCPAddr("tcp", host)
+	if err != nil {
+		return location{}
+	}
+
+	city, err := db.City(addr.IP)
+	if err != nil {
+		return location{}
+	}
+
+	return location{
+		Longitude: city.Location.Longitude,
+		Latitude:  city.Location.Latitude,
+		City:      city.City.Names["en"],
+		Country:   city.Country.IsoCode,
+		Continent: city.Continent.Code,
+	}
+}
+
+type loggingResponseWriter struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
+	return &loggingResponseWriter{w, http.StatusOK}
+}
+
+func (lrw *loggingResponseWriter) WriteHeader(code int) {
+	lrw.statusCode = code
+	lrw.ResponseWriter.WriteHeader(code)
+}

cmd/strelaypoolsrv/stats.go

@@ -0,0 +1,259 @@
+// Copyright (C) 2018 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"encoding/json"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+func init() {
+	processCollectorOpts := prometheus.ProcessCollectorOpts{
+		Namespace: "syncthing_relaypoolsrv",
+		PidFn: func() (int, error) {
+			return os.Getpid(), nil
+		},
+	}
+
+	prometheus.MustRegister(
+		prometheus.NewProcessCollector(processCollectorOpts),
+	)
+}
+
+var (
+	statusClient = http.Client{
+		Timeout: 5 * time.Second,
+	}
+
+	apiRequestsTotal   = makeCounter("api_requests_total", "Number of API requests.", "type", "result")
+	apiRequestsSeconds = makeSummary("api_requests_seconds", "Latency of API requests.", "type")
+
+	relayTestsTotal         = makeCounter("tests_total", "Number of relay tests.", "result")
+	relayTestActionsSeconds = makeSummary("test_actions_seconds", "Latency of relay test actions.", "type")
+
+	locationLookupSeconds = makeSummary("location_lookup_seconds", "Latency of location lookups.").WithLabelValues()
+
+	metricsRequestsSeconds = makeSummary("metrics_requests_seconds", "Latency of metric requests.").WithLabelValues()
+	scrapeSeconds          = makeSummary("relay_scrape_seconds", "Latency of metric scrapes from remote relays.", "result")
+
+	relayUptime             = makeGauge("relay_uptime", "Uptime of relay", "relay")
+	relayPendingSessionKeys = makeGauge("relay_pending_session_keys", "Number of pending session keys (two keys per session, one per each side of the connection)", "relay")
+	relayActiveSessions     = makeGauge("relay_active_sessions", "Number of sessions that are happening, a session contains two parties", "relay")
+	relayConnections        = makeGauge("relay_connections", "Number of devices connected to the relay", "relay")
+	relayProxies            = makeGauge("relay_proxies", "Number of active proxy routines sending data between peers (two proxies per session, one for each way)", "relay")
+	relayBytesProxied       = makeGauge("relay_bytes_proxied", "Number of bytes proxied by the relay", "relay")
+	relayGoRoutines         = makeGauge("relay_go_routines", "Number of Go routines in the process", "relay")
+	relaySessionRate        = makeGauge("relay_session_rate", "Rate applied per session", "relay")
+	relayGlobalRate         = makeGauge("relay_global_rate", "Global rate applied on the whole relay", "relay")
+	relayBuildInfo          = makeGauge("relay_build_info", "Build information about a relay", "relay", "go_version", "go_os", "go_arch")
+	relayLocationInfo       = makeGauge("relay_location_info", "Location information about a relay", "relay", "city", "country", "continent")
+
+	lastStats = make(map[string]stats)
+)
+
+func makeGauge(name string, help string, labels ...string) *prometheus.GaugeVec {
+	gauge := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "syncthing",
+			Subsystem: "relaypoolsrv",
+			Name:      name,
+			Help:      help,
+		},
+		labels,
+	)
+	prometheus.MustRegister(gauge)
+	return gauge
+}
+
+func makeSummary(name string, help string, labels ...string) *prometheus.SummaryVec {
+	summary := prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Namespace:  "syncthing",
+			Subsystem:  "relaypoolsrv",
+			Name:       name,
+			Help:       help,
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		labels,
+	)
+	prometheus.MustRegister(summary)
+	return summary
+}
+
+func makeCounter(name string, help string, labels ...string) *prometheus.CounterVec {
+	counter := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "relaypoolsrv",
+			Name:      name,
+			Help:      help,
+		},
+		labels,
+	)
+	prometheus.MustRegister(counter)
+	return counter
+}
+
+func statsRefresher(interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	for range ticker.C {
+		refreshStats()
+	}
+}
+
+type statsFetchResult struct {
+	relay *relay
+	stats *stats
+}
+
+func refreshStats() {
+	mut.RLock()
+	relays := append(permanentRelays, knownRelays...)
+	mut.RUnlock()
+
+	now := time.Now()
+	wg := sync.NewWaitGroup()
+
+	results := make(chan statsFetchResult, len(relays))
+	for _, rel := range relays {
+		wg.Add(1)
+		go func(rel *relay) {
+			t0 := time.Now()
+			stats := fetchStats(rel)
+			duration := time.Since(t0).Seconds()
+			result := "success"
+			if stats == nil {
+				result = "failed"
+			}
+			scrapeSeconds.WithLabelValues(result).Observe(duration)
+
+			results <- statsFetchResult{
+				relay: rel,
+				stats: fetchStats(rel),
+			}
+			wg.Done()
+		}(rel)
+	}
+
+	wg.Wait()
+	close(results)
+
+	mut.Lock()
+	relayBuildInfo.Reset()
+	relayLocationInfo.Reset()
+	for result := range results {
+		result.relay.StatsRetrieved = now
+		result.relay.Stats = result.stats
+		if result.stats == nil {
+			deleteMetrics(result.relay.uri.Host)
+		} else {
+			updateMetrics(result.relay.uri.Host, *result.stats, result.relay.Location)
+		}
+	}
+	mut.Unlock()
+}
+
+func fetchStats(relay *relay) *stats {
+	statusAddr := relay.uri.Query().Get("statusAddr")
+	if statusAddr == "" {
+		statusAddr = ":22070"
+	}
+
+	statusHost, statusPort, err := net.SplitHostPort(statusAddr)
+	if err != nil {
+		return nil
+	}
+
+	if statusHost == "" {
+		if host, _, err := net.SplitHostPort(relay.uri.Host); err != nil {
+			return nil
+		} else {
+			statusHost = host
+		}
+	}
+
+	url := "http://" + net.JoinHostPort(statusHost, statusPort) + "/status"
+
+	response, err := statusClient.Get(url)
+	if err != nil {
+		return nil
+	}
+
+	var stats stats
+
+	if json.NewDecoder(response.Body).Decode(&stats); err != nil {
+		return nil
+	}
+	return &stats
+}
+
+func updateMetrics(host string, stats stats, location location) {
+	if stats.GoVersion != "" || stats.GoOS != "" || stats.GoArch != "" {
+		relayBuildInfo.WithLabelValues(host, stats.GoVersion, stats.GoOS, stats.GoArch).Add(1)
+	}
+	if location.City != "" || location.Country != "" || location.Continent != "" {
+		relayLocationInfo.WithLabelValues(host, location.City, location.Country, location.Continent).Add(1)
+	}
+
+	if lastStat, ok := lastStats[host]; ok {
+		stats = mergeStats(stats, lastStat)
+	}
+
+	relayUptime.WithLabelValues(host).Set(float64(stats.UptimeSeconds))
+	relayPendingSessionKeys.WithLabelValues(host).Set(float64(stats.PendingSessionKeys))
+	relayActiveSessions.WithLabelValues(host).Set(float64(stats.ActiveSessions))
+	relayConnections.WithLabelValues(host).Set(float64(stats.Connections))
+	relayProxies.WithLabelValues(host).Set(float64(stats.Proxies))
+	relayBytesProxied.WithLabelValues(host).Set(float64(stats.BytesProxied))
+	relayGoRoutines.WithLabelValues(host).Set(float64(stats.GoRoutines))
+	relaySessionRate.WithLabelValues(host).Set(float64(stats.Options.SessionRate))
+	relayGlobalRate.WithLabelValues(host).Set(float64(stats.Options.GlobalRate))
+	lastStats[host] = stats
+}
+
+func deleteMetrics(host string) {
+	relayUptime.DeleteLabelValues(host)
+	relayPendingSessionKeys.DeleteLabelValues(host)
+	relayActiveSessions.DeleteLabelValues(host)
+	relayConnections.DeleteLabelValues(host)
+	relayProxies.DeleteLabelValues(host)
+	relayBytesProxied.DeleteLabelValues(host)
+	relayGoRoutines.DeleteLabelValues(host)
+	relaySessionRate.DeleteLabelValues(host)
+	relayGlobalRate.DeleteLabelValues(host)
+	delete(lastStats, host)
+}
+
+// Due to some unexplainable behaviour, some of the numbers sometimes travel slightly backwards (by less than 1%)
+// This happens between scrapes, which is 30s, so this can't be a race.
+// This causes prometheus to assume a "rate reset", hence causes phenomenal spikes.
+// One of the number that moves backwards is BytesProxied, which atomically increments a counter with numeric value
+// returned by net.Conn.Read(). I don't think that can return a negative value, so I have no idea what's going on.
+func mergeStats(new stats, old stats) stats {
+	new.UptimeSeconds = mergeValue(new.UptimeSeconds, old.UptimeSeconds)
+	new.PendingSessionKeys = mergeValue(new.PendingSessionKeys, old.PendingSessionKeys)
+	new.ActiveSessions = mergeValue(new.ActiveSessions, old.ActiveSessions)
+	new.Connections = mergeValue(new.Connections, old.Connections)
+	new.Proxies = mergeValue(new.Proxies, old.Proxies)
+	new.BytesProxied = mergeValue(new.BytesProxied, old.BytesProxied)
+	new.GoRoutines = mergeValue(new.GoRoutines, old.GoRoutines)
+	new.Options.SessionRate = mergeValue(new.Options.SessionRate, old.Options.SessionRate)
+	new.Options.GlobalRate = mergeValue(new.Options.GlobalRate, old.Options.GlobalRate)
+	return new
+}
+
+func mergeValue(new, old int) int {
+	if new >= old {
+		return new // normal increase
+	}
+	if float64(new) > 0.99*float64(old) {
+		return old // slight backward movement
+	}
+	return new // reset (relay restart)
+}

cmd/strelaypoolsrv/stats_test.go

@@ -0,0 +1,21 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"testing"
+)
+
+func TestMerge(t *testing.T) {
+	if mergeValue(1001, 1000) != 1001 {
+		t.Error("the computer says no")
+	}
+
+	if mergeValue(999, 1000) != 1000 {
+		t.Error("the computer says no")
+	}
+
+	if mergeValue(1, 1000) != 1 {
+		t.Error("the computer says no")
+	}
+}

cmd/strelaysrv/LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016 Sergey Kamardin
+Copyright (c) 2015 The Syncthing Project
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -18,4 +18,5 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.
+

cmd/strelaysrv/README.md

@@ -0,0 +1,131 @@
+strelaysrv
+==========
+
+This is the relay server for the `syncthing` project.
+
+:exclamation:Warnings:exclamation: - Read or regret
+-----
+
+By default, all relay servers will join to the default public relay pool, which means that the relay server will be available for public use, and **will consume your bandwidth** helping others to connect.
+
+If you wish to disable this behaviour, please specify the `-pools=""` argument.
+
+Please note that `strelaysrv` is only usable by `syncthing` **version v0.12 and onwards**.
+
+To run `strelaysrv` you need to have port 22067 available to the internet, which means you might need to port forward it and/or allow it through your firewall.
+
+Furthermore, by default `strelaysrv` will also expose a /status HTTP endpoint on port 22070, which is used by the pool servers to read metrics of the `strelaysrv`, such as  the current transfer rates, how many clients are connected, etc. If you wish this information to be available you may need to port forward and allow it through your firewall. This is not mandatory for the `strelaysrv` to function, and is used only to gather metrics and present them in the overview page of the pool server.
+
+At the point of writing the endpoint output looks as follows:
+
+```
+{
+    "bytesProxied": 0,
+    "goArch": "amd64",
+    "goMaxProcs": 1,
+    "goNumRoutine": 13,
+    "goOS": "linux",
+    "goVersion": "go1.6",
+    "kbps10s1m5m15m30m60m": [
+        0,
+        0,
+        0,
+        0,
+        0,
+        0
+    ],
+    "numActiveSessions": 0,
+    "numConnections": 0,
+    "numPendingSessionKeys": 2,
+    "numProxies": 0,
+    "options": {
+        "global-rate": 0,
+        "message-timeout": 60,
+        "network-timeout": 120,
+        "per-session-rate": 0,
+        "ping-interval": 60,
+        "pools": [
+            "https://relays.syncthing.net/endpoint"
+        ],
+        "provided-by": ""
+    },
+    "startTime": "2016-03-06T12:53:07.090847749-05:00",
+    "uptimeSeconds": 17
+}
+```
+
+If you wish to disable the /status endpoint, provide `-status-srv=""` as one of the arguments when starting the strelaysrv.
+
+Running for public use
+----
+Make sure you have a public IP with port 22067 open, or have forwarded port 22067 if you are behind a NAT.
+
+Run the `strelaysrv` with no arguments (or `-debug` if you want more output), and that should be enough for the server to join the public relay pool.
+You should see a message saying:
+```
+2015/09/21 22:45:46 pool.go:60: Joined https://relays.syncthing.net/endpoint rejoining in 48m0s
+```
+
+See `strelaysrv -help` for other options, such as rate limits, timeout intervals, etc.
+
+Running for private use
+-----
+
+Once you've started the `strelaysrv`, it will generate a key pair and print a URI:
+```bash
+relay://:22067/?id=EZQOIDM-6DDD4ZI-DJ65NSM-4OQWRAT-EIKSMJO-OZ552BO-WQZEGYY-STS5RQM&pingInterval=1m0s&networkTimeout=2m0s&sessionLimitBps=0&globalLimitBps=0&statusAddr=:22070
+```
+
+This URI contains a partial address of the relay server, as well as its options which in the future may be taken into account when choosing the most suitable relay.
+
+Because the `-listen` option was not used `strelaysrv` does not know its external IP, therefore you should replace the host part of the URI with your public IP address on which the `strelaysrv` will be available:
+
+```bash
+relay://192.0.2.1:22067/?id=EZQOIDM-6DDD4ZI-DJ65NSM-4OQWRAT-EIKSMJO-OZ552BO-WQZEGYY-STS5RQM&pingInterval=1m0s&networkTimeout=2m0s&sessionLimitBps=0&globalLimitBps=0&statusAddr=:22070
+```
+
+If you do not care about certificate pinning (improved security) or do not care about passing verbose settings to the clients, you can shorten the URL to just the host part:
+
+```bash
+relay://192.0.2.1:22067
+```
+
+This URI can then be used in `syncthing` clients as one of the relay servers by adding the URI to the "Sync Protocol Listen Address" field, under Actions and Settings.
+
+See `strelaysrv -help` for other options, such as rate limits, timeout intervals, etc.
+
+Other items available in this repo
+----
+##### testutil
+A test utility which can be used to test the connectivity of a relay server.
+You need to generate two x509 key pairs (key.pem and cert.pem), one for the client and one for the server, in separate directories.
+Afterwards, start the client:
+```bash
+./testutil -relay="relay://192.0.2.1:22067" -keys=certs/client/ -join
+```
+
+This prints out the client ID:
+```
+2015/09/21 23:00:52 main.go:42: ID: BG2C5ZA-W7XPFDO-LH222Z6-65F3HJX-ADFTGRT-3SBFIGM-KV26O2Q-E5RMRQ2
+```
+
+In the other terminal run the following:
+
+```bash
+ ./testutil -relay="relay://192.0.2.1:22067" -keys=certs/server/ -connect=BG2C5ZA-W7XPFDO-LH222Z6-65F3HJX-ADFTGRT-3SBFIGM-KV26O2Q-E5RMRQ2
+```
+
+Which should then give you an interactive prompt, where you can type things in one terminal, and they get relayed to the other terminal.
+
+Relay related libraries used by this repo
+----
+##### Relay protocol definition.
+
+[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/protocol)
+
+
+##### Relay client
+
+Only used by the testutil.
+
+[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/client)

cmd/strelaysrv/etc/firewall-ufw/strelaysrv

@@ -0,0 +1,9 @@
+[strelaysrv]
+title=Syncthing relay server
+description=Proxies traffic of syncthing client behind firewalls
+ports=22067/tcp
+
+[strelaysrv-metrics]
+title=Syncthing relay metrics
+description=Provides metrics about the syncthing relay server
+ports=22070/tcp

cmd/strelaysrv/etc/linux-systemd/default

@@ -0,0 +1,5 @@
+# Default settings for syncthing-relaysrv (strelaysrv).
+NAT=true
+
+## Add Options here:
+RELAYSRV_OPTS=

cmd/strelaysrv/etc/linux-systemd/strelaysrv.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=Syncthing Relay Server
+After=network.target
+Documentation=man:strelaysrv(1)
+
+[Service]
+WorkingDirectory=/var/lib/syncthing-relaysrv
+EnvironmentFile=/etc/default/syncthing-relaysrv
+ExecStart=/usr/bin/strelaysrv -nat=${NAT} $RELAYSRV_OPTS
+
+# Hardening
+User=syncthing-relaysrv
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-relaysrv
+NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
+
+[Install]
+WantedBy=multi-user.target
+Alias=syncthing-relaysrv.service

cmd/strelaysrv/listener.go

@@ -0,0 +1,357 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors.
+
+package main
+
+import (
+	"crypto/tls"
+	"encoding/hex"
+	"log"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/tlsutil"
+
+	"github.com/syncthing/syncthing/lib/relay/protocol"
+)
+
+var (
+	outboxesMut    = sync.RWMutex{}
+	outboxes       = make(map[syncthingprotocol.DeviceID]chan interface{})
+	numConnections int64
+)
+
+func listener(proto, addr string, config *tls.Config) {
+	tcpListener, err := net.Listen("tcp", addr)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	listener := tlsutil.DowngradingListener{
+		Listener: tcpListener,
+	}
+
+	for {
+		conn, isTLS, err := listener.AcceptNoWrapTLS()
+		if err != nil {
+			if debug {
+				log.Println("Listener failed to accept connection from", conn.RemoteAddr(), ". Possibly a TCP Ping.")
+			}
+			continue
+		}
+
+		setTCPOptions(conn)
+
+		if debug {
+			log.Println("Listener accepted connection from", conn.RemoteAddr(), "tls", isTLS)
+		}
+
+		if isTLS {
+			go protocolConnectionHandler(conn, config)
+		} else {
+			go sessionConnectionHandler(conn)
+		}
+
+	}
+}
+
+func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
+	conn := tls.Server(tcpConn, config)
+	if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil {
+		if debug {
+			log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr())
+		}
+		conn.Close()
+		return
+	}
+	err := conn.Handshake()
+	if err != nil {
+		if debug {
+			log.Println("Protocol connection TLS handshake:", conn.RemoteAddr(), err)
+		}
+		conn.Close()
+		return
+	}
+
+	state := conn.ConnectionState()
+	if (!state.NegotiatedProtocolIsMutual || state.NegotiatedProtocol != protocol.ProtocolName) && debug {
+		log.Println("Protocol negotiation error")
+	}
+
+	certs := state.PeerCertificates
+	if len(certs) != 1 {
+		if debug {
+			log.Println("Certificate list error")
+		}
+		conn.Close()
+		return
+	}
+	conn.SetDeadline(time.Time{})
+
+	id := syncthingprotocol.NewDeviceID(certs[0].Raw)
+
+	messages := make(chan interface{})
+	errors := make(chan error, 1)
+	outbox := make(chan interface{})
+
+	// Read messages from the connection and send them on the messages
+	// channel. When there is an error, send it on the error channel and
+	// return. Applies also when the connection gets closed, so the pattern
+	// below is to close the connection on error, then wait for the error
+	// signal from messageReader to exit.
+	go messageReader(conn, messages, errors)
+
+	pingTicker := time.NewTicker(pingInterval)
+	defer pingTicker.Stop()
+	timeoutTicker := time.NewTimer(networkTimeout)
+	defer timeoutTicker.Stop()
+	joined := false
+
+	for {
+		select {
+		case message := <-messages:
+			timeoutTicker.Reset(networkTimeout)
+			if debug {
+				log.Printf("Message %T from %s", message, id)
+			}
+
+			switch msg := message.(type) {
+			case protocol.JoinRelayRequest:
+				if atomic.LoadInt32(&overLimit) > 0 {
+					protocol.WriteMessage(conn, protocol.RelayFull{})
+					if debug {
+						log.Println("Refusing join request from", id, "due to being over limits")
+					}
+					conn.Close()
+					limitCheckTimer.Reset(time.Second)
+					continue
+				}
+
+				outboxesMut.RLock()
+				_, ok := outboxes[id]
+				outboxesMut.RUnlock()
+				if ok {
+					protocol.WriteMessage(conn, protocol.ResponseAlreadyConnected)
+					if debug {
+						log.Println("Already have a peer with the same ID", id, conn.RemoteAddr())
+					}
+					conn.Close()
+					continue
+				}
+
+				outboxesMut.Lock()
+				outboxes[id] = outbox
+				outboxesMut.Unlock()
+				joined = true
+
+				protocol.WriteMessage(conn, protocol.ResponseSuccess)
+
+			case protocol.ConnectRequest:
+				requestedPeer := syncthingprotocol.DeviceIDFromBytes(msg.ID)
+				outboxesMut.RLock()
+				peerOutbox, ok := outboxes[requestedPeer]
+				outboxesMut.RUnlock()
+				if !ok {
+					if debug {
+						log.Println(id, "is looking for", requestedPeer, "which does not exist")
+					}
+					protocol.WriteMessage(conn, protocol.ResponseNotFound)
+					conn.Close()
+					continue
+				}
+				// requestedPeer is the server, id is the client
+				ses := newSession(requestedPeer, id, sessionLimiter, globalLimiter)
+
+				go ses.Serve()
+
+				clientInvitation := ses.GetClientInvitationMessage()
+				serverInvitation := ses.GetServerInvitationMessage()
+
+				if err := protocol.WriteMessage(conn, clientInvitation); err != nil {
+					if debug {
+						log.Printf("Error sending invitation from %s to client: %s", id, err)
+					}
+					conn.Close()
+					continue
+				}
+
+				select {
+				case peerOutbox <- serverInvitation:
+					if debug {
+						log.Println("Sent invitation from", id, "to", requestedPeer)
+					}
+				case <-time.After(time.Second):
+					if debug {
+						log.Println("Could not send invitation from", id, "to", requestedPeer, "as peer disconnected")
+					}
+
+				}
+				conn.Close()
+
+			case protocol.Ping:
+				if err := protocol.WriteMessage(conn, protocol.Pong{}); err != nil {
+					if debug {
+						log.Println("Error writing pong:", err)
+					}
+					conn.Close()
+					continue
+				}
+
+			case protocol.Pong:
+				// Nothing
+
+			default:
+				if debug {
+					log.Printf("Unknown message %s: %T", id, message)
+				}
+				protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage)
+				conn.Close()
+			}
+
+		case err := <-errors:
+			if debug {
+				log.Printf("Closing connection %s: %s", id, err)
+			}
+
+			// Potentially closing a second time.
+			conn.Close()
+
+			if joined {
+				// Only delete the outbox if the client is joined, as it might be
+				// a lookup request coming from the same client.
+				outboxesMut.Lock()
+				delete(outboxes, id)
+				outboxesMut.Unlock()
+				// Also, kill all sessions related to this node, as it probably
+				// went offline. This is for the other end to realize the client
+				// is no longer there faster. This also helps resolve
+				// 'already connected' errors when one of the sides is
+				// restarting, and connecting to the other peer before the other
+				// peer even realised that the node has gone away.
+				dropSessions(id)
+			}
+			return
+
+		case <-pingTicker.C:
+			if !joined {
+				if debug {
+					log.Println(id, "didn't join within", pingInterval)
+				}
+				conn.Close()
+				continue
+			}
+
+			if err := protocol.WriteMessage(conn, protocol.Ping{}); err != nil {
+				if debug {
+					log.Println(id, err)
+				}
+				conn.Close()
+			}
+
+			if atomic.LoadInt32(&overLimit) > 0 && !hasSessions(id) {
+				if debug {
+					log.Println("Dropping", id, "as it has no sessions and we are over our limits")
+				}
+				protocol.WriteMessage(conn, protocol.RelayFull{})
+				conn.Close()
+
+				limitCheckTimer.Reset(time.Second)
+			}
+
+		case <-timeoutTicker.C:
+			// We should receive a error from the reader loop, which will cause
+			// us to quit this loop.
+			if debug {
+				log.Printf("%s timed out", id)
+			}
+			conn.Close()
+
+		case msg := <-outbox:
+			if debug {
+				log.Printf("Sending message %T to %s", msg, id)
+			}
+			if err := protocol.WriteMessage(conn, msg); err != nil {
+				if debug {
+					log.Println(id, err)
+				}
+				conn.Close()
+			}
+		}
+	}
+}
+
+func sessionConnectionHandler(conn net.Conn) {
+	if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil {
+		if debug {
+			log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr())
+		}
+		conn.Close()
+		return
+	}
+
+	message, err := protocol.ReadMessage(conn)
+	if err != nil {
+		return
+	}
+
+	switch msg := message.(type) {
+	case protocol.JoinSessionRequest:
+		ses := findSession(string(msg.Key))
+		if debug {
+			log.Println(conn.RemoteAddr(), "session lookup", ses, hex.EncodeToString(msg.Key)[:5])
+		}
+
+		if ses == nil {
+			protocol.WriteMessage(conn, protocol.ResponseNotFound)
+			conn.Close()
+			return
+		}
+
+		if !ses.AddConnection(conn) {
+			if debug {
+				log.Println("Failed to add", conn.RemoteAddr(), "to session", ses)
+			}
+			protocol.WriteMessage(conn, protocol.ResponseAlreadyConnected)
+			conn.Close()
+			return
+		}
+
+		if err := protocol.WriteMessage(conn, protocol.ResponseSuccess); err != nil {
+			if debug {
+				log.Println("Failed to send session join response to ", conn.RemoteAddr(), "for", ses)
+			}
+			return
+		}
+
+		if err := conn.SetDeadline(time.Time{}); err != nil {
+			if debug {
+				log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr())
+			}
+			conn.Close()
+			return
+		}
+
+	default:
+		if debug {
+			log.Println("Unexpected message from", conn.RemoteAddr(), message)
+		}
+		protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage)
+		conn.Close()
+	}
+}
+
+func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) {
+	atomic.AddInt64(&numConnections, 1)
+	defer atomic.AddInt64(&numConnections, -1)
+
+	for {
+		msg, err := protocol.ReadMessage(conn)
+		if err != nil {
+			errors <- err
+			return
+		}
+		messages <- msg
+	}
+}

cmd/strelaysrv/main.go

@@ -0,0 +1,301 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors.
+
+package main
+
+import (
+	"crypto/tls"
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/relay/protocol"
+	"github.com/syncthing/syncthing/lib/tlsutil"
+	"golang.org/x/time/rate"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/nat"
+	_ "github.com/syncthing/syncthing/lib/pmp"
+	_ "github.com/syncthing/syncthing/lib/upnp"
+
+	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
+)
+
+var (
+	listen string
+	debug  bool
+
+	sessionAddress []byte
+	sessionPort    uint16
+
+	networkTimeout = 2 * time.Minute
+	pingInterval   = time.Minute
+	messageTimeout = time.Minute
+
+	limitCheckTimer *time.Timer
+
+	sessionLimitBps   int
+	globalLimitBps    int
+	overLimit         int32
+	descriptorLimit   int64
+	sessionLimiter    *rate.Limiter
+	globalLimiter     *rate.Limiter
+	networkBufferSize int
+
+	statusAddr       string
+	poolAddrs        string
+	pools            []string
+	providedBy       string
+	defaultPoolAddrs = "https://relays.syncthing.net/endpoint"
+
+	natEnabled bool
+	natLease   int
+	natRenewal int
+	natTimeout int
+
+	pprofEnabled bool
+)
+
+// httpClient is the HTTP client we use for outbound requests. It has a
+// timeout and may get further options set during initialization.
+var httpClient = &http.Client{
+	Timeout: 30 * time.Second,
+}
+
+func main() {
+	log.SetFlags(log.Lshortfile | log.LstdFlags)
+
+	var dir, extAddress, proto string
+
+	flag.StringVar(&listen, "listen", ":22067", "Protocol listen address")
+	flag.StringVar(&dir, "keys", ".", "Directory where cert.pem and key.pem is stored")
+	flag.DurationVar(&networkTimeout, "network-timeout", networkTimeout, "Timeout for network operations between the client and the relay.\n\tIf no data is received between the client and the relay in this period of time, the connection is terminated.\n\tFurthermore, if no data is sent between either clients being relayed within this period of time, the session is also terminated.")
+	flag.DurationVar(&pingInterval, "ping-interval", pingInterval, "How often pings are sent")
+	flag.DurationVar(&messageTimeout, "message-timeout", messageTimeout, "Maximum amount of time we wait for relevant messages to arrive")
+	flag.IntVar(&sessionLimitBps, "per-session-rate", sessionLimitBps, "Per session rate limit, in bytes/s")
+	flag.IntVar(&globalLimitBps, "global-rate", globalLimitBps, "Global rate limit, in bytes/s")
+	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
+	flag.StringVar(&statusAddr, "status-srv", ":22070", "Listen address for status service (blank to disable)")
+	flag.StringVar(&poolAddrs, "pools", defaultPoolAddrs, "Comma separated list of relay pool addresses to join")
+	flag.StringVar(&providedBy, "provided-by", "", "An optional description about who provides the relay")
+	flag.StringVar(&extAddress, "ext-address", "", "An optional address to advertise as being available on.\n\tAllows listening on an unprivileged port with port forwarding from e.g. 443, and be connected to on port 443.")
+	flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
+	flag.BoolVar(&natEnabled, "nat", false, "Use UPnP/NAT-PMP to acquire external port mapping")
+	flag.IntVar(&natLease, "nat-lease", 60, "NAT lease length in minutes")
+	flag.IntVar(&natRenewal, "nat-renewal", 30, "NAT renewal frequency in minutes")
+	flag.IntVar(&natTimeout, "nat-timeout", 10, "NAT discovery timeout in seconds")
+	flag.BoolVar(&pprofEnabled, "pprof", false, "Enable the built in profiling on the status server")
+	flag.IntVar(&networkBufferSize, "network-buffer", 2048, "Network buffer size (two of these per proxied connection)")
+	showVersion := flag.Bool("version", false, "Show version")
+	flag.Parse()
+
+	if *showVersion {
+		fmt.Println(build.LongVersion)
+		return
+	}
+
+	if extAddress == "" {
+		extAddress = listen
+	}
+
+	if len(providedBy) > 30 {
+		log.Fatal("Provided-by cannot be longer than 30 characters")
+	}
+
+	addr, err := net.ResolveTCPAddr(proto, extAddress)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	laddr, err := net.ResolveTCPAddr(proto, listen)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if laddr.IP != nil && !laddr.IP.IsUnspecified() {
+		// We bind to a specific address. Our outgoing HTTP requests should
+		// also come from that address.
+		laddr.Port = 0
+		boundDialer := &net.Dialer{LocalAddr: laddr}
+		httpClient.Transport = &http.Transport{
+			DialContext: boundDialer.DialContext,
+		}
+	}
+
+	log.Println(build.LongVersion)
+
+	maxDescriptors, err := osutil.MaximizeOpenFileLimit()
+	if maxDescriptors > 0 {
+		// Assume that 20% of FD's are leaked/unaccounted for.
+		descriptorLimit = int64(maxDescriptors*80) / 100
+		log.Println("Connection limit", descriptorLimit)
+
+		go monitorLimits()
+	} else if err != nil && runtime.GOOS != "windows" {
+		log.Println("Assuming no connection limit, due to error retrieving rlimits:", err)
+	}
+
+	sessionAddress = addr.IP[:]
+	sessionPort = uint16(addr.Port)
+
+	certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem")
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		log.Println("Failed to load keypair. Generating one, this might take a while...")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv")
+		if err != nil {
+			log.Fatalln("Failed to generate X509 key pair:", err)
+		}
+	}
+
+	tlsCfg := &tls.Config{
+		Certificates:           []tls.Certificate{cert},
+		NextProtos:             []string{protocol.ProtocolName},
+		ClientAuth:             tls.RequestClientCert,
+		SessionTicketsDisabled: true,
+		InsecureSkipVerify:     true,
+		MinVersion:             tls.VersionTLS12,
+		CipherSuites: []uint16{
+			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+		},
+	}
+
+	id := syncthingprotocol.NewDeviceID(cert.Certificate[0])
+	if debug {
+		log.Println("ID:", id)
+	}
+
+	wrapper := config.Wrap("config", config.New(id), events.NoopLogger)
+	wrapper.SetOptions(config.OptionsConfiguration{
+		NATLeaseM:   natLease,
+		NATRenewalM: natRenewal,
+		NATTimeoutS: natTimeout,
+	})
+	natSvc := nat.NewService(id, wrapper)
+	mapping := mapping{natSvc.NewMapping(nat.TCP, addr.IP, addr.Port)}
+
+	if natEnabled {
+		go natSvc.Serve()
+		found := make(chan struct{})
+		mapping.OnChanged(func(_ *nat.Mapping, _, _ []nat.Address) {
+			select {
+			case found <- struct{}{}:
+			default:
+			}
+		})
+
+		// Need to wait a few extra seconds, since NAT library waits exactly natTimeout seconds on all interfaces.
+		timeout := time.Duration(natTimeout+2) * time.Second
+		log.Printf("Waiting %s to acquire NAT mapping", timeout)
+
+		select {
+		case <-found:
+			log.Printf("Found NAT mapping: %s", mapping.ExternalAddresses())
+		case <-time.After(timeout):
+			log.Println("Timeout out waiting for NAT mapping.")
+		}
+	}
+
+	if sessionLimitBps > 0 {
+		sessionLimiter = rate.NewLimiter(rate.Limit(sessionLimitBps), 2*sessionLimitBps)
+	}
+	if globalLimitBps > 0 {
+		globalLimiter = rate.NewLimiter(rate.Limit(globalLimitBps), 2*globalLimitBps)
+	}
+
+	if statusAddr != "" {
+		go statusService(statusAddr)
+	}
+
+	uri, err := url.Parse(fmt.Sprintf("relay://%s/?id=%s&pingInterval=%s&networkTimeout=%s&sessionLimitBps=%d&globalLimitBps=%d&statusAddr=%s&providedBy=%s", mapping.Address(), id, pingInterval, networkTimeout, sessionLimitBps, globalLimitBps, statusAddr, providedBy))
+	if err != nil {
+		log.Fatalln("Failed to construct URI", err)
+	}
+
+	log.Println("URI:", uri.String())
+
+	if poolAddrs == defaultPoolAddrs {
+		log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+		log.Println("!!  Joining default relay pools, this relay will be available for public use. !!")
+		log.Println(`!!      Use the -pools="" command line option to make the relay private.      !!`)
+		log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+	}
+
+	pools = strings.Split(poolAddrs, ",")
+	for _, pool := range pools {
+		pool = strings.TrimSpace(pool)
+		if len(pool) > 0 {
+			go poolHandler(pool, uri, mapping)
+		}
+	}
+
+	go listener(proto, listen, tlsCfg)
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+	<-sigs
+
+	// Gracefully close all connections, hoping that clients will be faster
+	// to realize that the relay is now gone.
+
+	sessionMut.RLock()
+	for _, session := range activeSessions {
+		session.CloseConns()
+	}
+
+	for _, session := range pendingSessions {
+		session.CloseConns()
+	}
+	sessionMut.RUnlock()
+
+	outboxesMut.RLock()
+	for _, outbox := range outboxes {
+		close(outbox)
+	}
+	outboxesMut.RUnlock()
+
+	time.Sleep(500 * time.Millisecond)
+}
+
+func monitorLimits() {
+	limitCheckTimer = time.NewTimer(time.Minute)
+	for range limitCheckTimer.C {
+		if atomic.LoadInt64(&numConnections)+atomic.LoadInt64(&numProxies) > descriptorLimit {
+			atomic.StoreInt32(&overLimit, 1)
+			log.Println("Gone past our connection limits. Starting to refuse new/drop idle connections.")
+		} else if atomic.CompareAndSwapInt32(&overLimit, 1, 0) {
+			log.Println("Dropped below our connection limits. Accepting new connections.")
+		}
+		limitCheckTimer.Reset(time.Minute)
+	}
+}
+
+type mapping struct {
+	*nat.Mapping
+}
+
+func (m *mapping) Address() nat.Address {
+	ext := m.ExternalAddresses()
+	if len(ext) > 0 {
+		return ext[0]
+	}
+	return m.Mapping.Address()
+}

cmd/strelaysrv/pool.go

@@ -0,0 +1,65 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"net/url"
+	"time"
+)
+
+func poolHandler(pool string, uri *url.URL, mapping mapping) {
+	if debug {
+		log.Println("Joining", pool)
+	}
+	for {
+		uriCopy := *uri
+		uriCopy.Host = mapping.Address().String()
+
+		var b bytes.Buffer
+		json.NewEncoder(&b).Encode(struct {
+			URL string `json:"url"`
+		}{
+			uriCopy.String(),
+		})
+
+		resp, err := httpClient.Post(pool, "application/json", &b)
+		if err != nil {
+			log.Println("Error joining pool", pool, err)
+		} else if resp.StatusCode == 500 {
+			bs, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				log.Println("Failed to join", pool, "due to an internal server error. Could not read response:", err)
+			} else {
+				log.Println("Failed to join", pool, "due to an internal server error:", string(bs))
+			}
+			resp.Body.Close()
+		} else if resp.StatusCode == 429 {
+			log.Println(pool, "under load, will retry in a minute")
+			time.Sleep(time.Minute)
+			continue
+		} else if resp.StatusCode == 401 {
+			log.Println(pool, "failed to join due to IP address not matching external address. Aborting")
+			return
+		} else if resp.StatusCode == 200 {
+			var x struct {
+				EvictionIn time.Duration `json:"evictionIn"`
+			}
+			err := json.NewDecoder(resp.Body).Decode(&x)
+			if err == nil {
+				rejoin := x.EvictionIn - (x.EvictionIn / 5)
+				log.Println("Joined", pool, "rejoining in", rejoin)
+				time.Sleep(rejoin)
+				continue
+			} else {
+				log.Println("Failed to deserialize response", err)
+			}
+		} else {
+			log.Println(pool, "unknown response type from server", resp.StatusCode)
+		}
+		time.Sleep(time.Hour)
+	}
+}

cmd/strelaysrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-relaysrv --ingroup syncthing syncthing-relaysrv

cmd/strelaysrv/session.go

@@ -0,0 +1,353 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors.
+
+package main
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"log"
+	"math"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"golang.org/x/time/rate"
+
+	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/relay/protocol"
+)
+
+var (
+	sessionMut      = sync.RWMutex{}
+	activeSessions  = make([]*session, 0)
+	pendingSessions = make(map[string]*session)
+	numProxies      int64
+	bytesProxied    int64
+)
+
+func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionRateLimit, globalRateLimit *rate.Limiter) *session {
+	serverkey := make([]byte, 32)
+	_, err := rand.Read(serverkey)
+	if err != nil {
+		return nil
+	}
+
+	clientkey := make([]byte, 32)
+	_, err = rand.Read(clientkey)
+	if err != nil {
+		return nil
+	}
+
+	ses := &session{
+		serverkey: serverkey,
+		serverid:  serverid,
+		clientkey: clientkey,
+		clientid:  clientid,
+		rateLimit: makeRateLimitFunc(sessionRateLimit, globalRateLimit),
+		connsChan: make(chan net.Conn),
+		conns:     make([]net.Conn, 0, 2),
+	}
+
+	if debug {
+		log.Println("New session", ses)
+	}
+
+	sessionMut.Lock()
+	pendingSessions[string(ses.serverkey)] = ses
+	pendingSessions[string(ses.clientkey)] = ses
+	sessionMut.Unlock()
+
+	return ses
+}
+
+func findSession(key string) *session {
+	sessionMut.Lock()
+	defer sessionMut.Unlock()
+	ses, ok := pendingSessions[key]
+	if !ok {
+		return nil
+
+	}
+	delete(pendingSessions, key)
+	return ses
+}
+
+func dropSessions(id syncthingprotocol.DeviceID) {
+	sessionMut.RLock()
+	for _, session := range activeSessions {
+		if session.HasParticipant(id) {
+			if debug {
+				log.Println("Dropping session", session, "involving", id)
+			}
+			session.CloseConns()
+		}
+	}
+	sessionMut.RUnlock()
+}
+
+func hasSessions(id syncthingprotocol.DeviceID) bool {
+	sessionMut.RLock()
+	has := false
+	for _, session := range activeSessions {
+		if session.HasParticipant(id) {
+			has = true
+			break
+		}
+	}
+	sessionMut.RUnlock()
+	return has
+}
+
+type session struct {
+	mut sync.Mutex
+
+	serverkey []byte
+	serverid  syncthingprotocol.DeviceID
+
+	clientkey []byte
+	clientid  syncthingprotocol.DeviceID
+
+	rateLimit func(bytes int)
+
+	connsChan chan net.Conn
+	conns     []net.Conn
+}
+
+func (s *session) AddConnection(conn net.Conn) bool {
+	if debug {
+		log.Println("New connection for", s, "from", conn.RemoteAddr())
+	}
+
+	select {
+	case s.connsChan <- conn:
+		return true
+	default:
+	}
+	return false
+}
+
+func (s *session) Serve() {
+	timedout := time.After(messageTimeout)
+
+	if debug {
+		log.Println("Session", s, "serving")
+	}
+
+	for {
+		select {
+		case conn := <-s.connsChan:
+			s.mut.Lock()
+			s.conns = append(s.conns, conn)
+			s.mut.Unlock()
+			// We're the only ones mutating s.conns, hence we are free to read it.
+			if len(s.conns) < 2 {
+				continue
+			}
+
+			close(s.connsChan)
+
+			if debug {
+				log.Println("Session", s, "starting between", s.conns[0].RemoteAddr(), "and", s.conns[1].RemoteAddr())
+			}
+
+			wg := sync.WaitGroup{}
+			wg.Add(2)
+
+			var err0 error
+			go func() {
+				err0 = s.proxy(s.conns[0], s.conns[1])
+				wg.Done()
+			}()
+
+			var err1 error
+			go func() {
+				err1 = s.proxy(s.conns[1], s.conns[0])
+				wg.Done()
+			}()
+
+			sessionMut.Lock()
+			activeSessions = append(activeSessions, s)
+			sessionMut.Unlock()
+
+			wg.Wait()
+
+			if debug {
+				log.Println("Session", s, "ended, outcomes:", err0, "and", err1)
+			}
+			goto done
+
+		case <-timedout:
+			if debug {
+				log.Println("Session", s, "timed out")
+			}
+			goto done
+		}
+	}
+done:
+	// We can end up here in 3 cases:
+	// 1. Timeout joining, in which case there are potentially entries in pendingSessions
+	// 2. General session end/timeout, in which case there are entries in activeSessions
+	// 3. Protocol handler calls dropSession as one of its clients disconnects.
+
+	sessionMut.Lock()
+	delete(pendingSessions, string(s.serverkey))
+	delete(pendingSessions, string(s.clientkey))
+
+	for i, session := range activeSessions {
+		if session == s {
+			l := len(activeSessions) - 1
+			activeSessions[i] = activeSessions[l]
+			activeSessions[l] = nil
+			activeSessions = activeSessions[:l]
+		}
+	}
+	sessionMut.Unlock()
+
+	// If we are here because of case 2 or 3, we are potentially closing some or
+	// all connections a second time.
+	s.CloseConns()
+
+	if debug {
+		log.Println("Session", s, "stopping")
+	}
+}
+
+func (s *session) GetClientInvitationMessage() protocol.SessionInvitation {
+	return protocol.SessionInvitation{
+		From:         s.serverid[:],
+		Key:          s.clientkey,
+		Address:      sessionAddress,
+		Port:         sessionPort,
+		ServerSocket: false,
+	}
+}
+
+func (s *session) GetServerInvitationMessage() protocol.SessionInvitation {
+	return protocol.SessionInvitation{
+		From:         s.clientid[:],
+		Key:          s.serverkey,
+		Address:      sessionAddress,
+		Port:         sessionPort,
+		ServerSocket: true,
+	}
+}
+
+func (s *session) HasParticipant(id syncthingprotocol.DeviceID) bool {
+	return s.clientid == id || s.serverid == id
+}
+
+func (s *session) CloseConns() {
+	s.mut.Lock()
+	for _, conn := range s.conns {
+		conn.Close()
+	}
+	s.mut.Unlock()
+}
+
+func (s *session) proxy(c1, c2 net.Conn) error {
+	if debug {
+		log.Println("Proxy", c1.RemoteAddr(), "->", c2.RemoteAddr())
+	}
+
+	atomic.AddInt64(&numProxies, 1)
+	defer atomic.AddInt64(&numProxies, -1)
+
+	buf := make([]byte, networkBufferSize)
+	for {
+		c1.SetReadDeadline(time.Now().Add(networkTimeout))
+		n, err := c1.Read(buf)
+		if err != nil {
+			return err
+		}
+
+		atomic.AddInt64(&bytesProxied, int64(n))
+
+		if debug {
+			log.Printf("%d bytes from %s to %s", n, c1.RemoteAddr(), c2.RemoteAddr())
+		}
+
+		if s.rateLimit != nil {
+			s.rateLimit(n)
+		}
+
+		c2.SetWriteDeadline(time.Now().Add(networkTimeout))
+		_, err = c2.Write(buf[:n])
+		if err != nil {
+			return err
+		}
+	}
+}
+
+func (s *session) String() string {
+	return fmt.Sprintf("<%s/%s>", hex.EncodeToString(s.clientkey)[:5], hex.EncodeToString(s.serverkey)[:5])
+}
+
+func makeRateLimitFunc(sessionRateLimit, globalRateLimit *rate.Limiter) func(int) {
+	// This may be a case of super duper premature optimization... We build an
+	// optimized function to do the rate limiting here based on what we need
+	// to do and then use it in the loop.
+
+	if sessionRateLimit == nil && globalRateLimit == nil {
+		// No limiting needed. We could equally well return a func(int64){} and
+		// not do a nil check were we use it, but I think the nil check there
+		// makes it clear that there will be no limiting if none is
+		// configured...
+		return nil
+	}
+
+	if sessionRateLimit == nil {
+		// We only have a global limiter
+		return func(bytes int) {
+			take(bytes, globalRateLimit)
+		}
+	}
+
+	if globalRateLimit == nil {
+		// We only have a session limiter
+		return func(bytes int) {
+			take(bytes, sessionRateLimit)
+		}
+	}
+
+	// We have both. Queue the bytes on both the global and session specific
+	// rate limiters.
+	return func(bytes int) {
+		take(bytes, sessionRateLimit, globalRateLimit)
+	}
+}
+
+// take is a utility function to consume tokens from a set of rate.Limiters.
+// Tokens are consumed in parallel on all limiters, respecting their
+// individual burst sizes.
+func take(tokens int, ls ...*rate.Limiter) {
+	// minBurst is the smallest burst size supported by all limiters.
+	minBurst := int(math.MaxInt32)
+	for _, l := range ls {
+		if burst := l.Burst(); burst < minBurst {
+			minBurst = burst
+		}
+	}
+
+	for tokens > 0 {
+		// chunk is how many tokens we can consume at a time
+		chunk := tokens
+		if chunk > minBurst {
+			chunk = minBurst
+		}
+
+		// maxDelay is the longest delay mandated by any of the limiters for
+		// the chosen chunk size.
+		var maxDelay time.Duration
+		for _, l := range ls {
+			res := l.ReserveN(time.Now(), chunk)
+			if del := res.Delay(); del > maxDelay {
+				maxDelay = del
+			}
+		}
+
+		time.Sleep(maxDelay)
+		tokens -= chunk
+	}
+}

cmd/strelaysrv/status.go

@@ -0,0 +1,129 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors.
+
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+	"net/http/pprof"
+	"runtime"
+	"sync/atomic"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/build"
+)
+
+var rc *rateCalculator
+
+func statusService(addr string) {
+	rc = newRateCalculator(360, 10*time.Second, &bytesProxied)
+
+	handler := http.NewServeMux()
+	handler.HandleFunc("/status", getStatus)
+	if pprofEnabled {
+		handler.HandleFunc("/debug/pprof/", pprof.Index)
+	}
+
+	srv := http.Server{
+		Addr:        addr,
+		Handler:     handler,
+		ReadTimeout: 15 * time.Second,
+	}
+	srv.SetKeepAlivesEnabled(false)
+	if err := srv.ListenAndServe(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func getStatus(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	status := make(map[string]interface{})
+
+	sessionMut.Lock()
+	// This can potentially be double the number of pending sessions, as each session has two keys, one for each side.
+	status["version"] = build.Version
+	status["buildHost"] = build.Host
+	status["buildUser"] = build.User
+	status["buildDate"] = build.Date
+	status["startTime"] = rc.startTime
+	status["uptimeSeconds"] = time.Since(rc.startTime) / time.Second
+	status["numPendingSessionKeys"] = len(pendingSessions)
+	status["numActiveSessions"] = len(activeSessions)
+	sessionMut.Unlock()
+	status["numConnections"] = atomic.LoadInt64(&numConnections)
+	status["numProxies"] = atomic.LoadInt64(&numProxies)
+	status["bytesProxied"] = atomic.LoadInt64(&bytesProxied)
+	status["goVersion"] = runtime.Version()
+	status["goOS"] = runtime.GOOS
+	status["goArch"] = runtime.GOARCH
+	status["goMaxProcs"] = runtime.GOMAXPROCS(-1)
+	status["goNumRoutine"] = runtime.NumGoroutine()
+	status["kbps10s1m5m15m30m60m"] = []int64{
+		rc.rate(1) * 8 / 1000, // each interval is 10s
+		rc.rate(60/10) * 8 / 1000,
+		rc.rate(5*60/10) * 8 / 1000,
+		rc.rate(15*60/10) * 8 / 1000,
+		rc.rate(30*60/10) * 8 / 1000,
+		rc.rate(60*60/10) * 8 / 1000,
+	}
+	status["options"] = map[string]interface{}{
+		"network-timeout":  networkTimeout / time.Second,
+		"ping-interval":    pingInterval / time.Second,
+		"message-timeout":  messageTimeout / time.Second,
+		"per-session-rate": sessionLimitBps,
+		"global-rate":      globalLimitBps,
+		"pools":            pools,
+		"provided-by":      providedBy,
+	}
+
+	bs, err := json.MarshalIndent(status, "", "    ")
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bs)
+}
+
+type rateCalculator struct {
+	counter   *int64 // atomic, must remain 64-bit aligned
+	rates     []int64
+	prev      int64
+	startTime time.Time
+}
+
+func newRateCalculator(keepIntervals int, interval time.Duration, counter *int64) *rateCalculator {
+	r := &rateCalculator{
+		rates:     make([]int64, keepIntervals),
+		counter:   counter,
+		startTime: time.Now(),
+	}
+
+	go r.updateRates(interval)
+
+	return r
+}
+
+func (r *rateCalculator) updateRates(interval time.Duration) {
+	for {
+		now := time.Now()
+		next := now.Truncate(interval).Add(interval)
+		time.Sleep(next.Sub(now))
+
+		cur := atomic.LoadInt64(r.counter)
+		rate := int64(float64(cur-r.prev) / interval.Seconds())
+		copy(r.rates[1:], r.rates)
+		r.rates[0] = rate
+		r.prev = cur
+	}
+}
+
+func (r *rateCalculator) rate(periods int) int64 {
+	var tot int64
+	for i := 0; i < periods; i++ {
+		tot += r.rates[i]
+	}
+	return tot / int64(periods)
+}

cmd/strelaysrv/testutil/main.go

@@ -0,0 +1,152 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"bufio"
+	"crypto/tls"
+	"flag"
+	"log"
+	"net"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/relay/client"
+	"github.com/syncthing/syncthing/lib/relay/protocol"
+)
+
+func main() {
+	log.SetOutput(os.Stdout)
+	log.SetFlags(log.LstdFlags | log.Lshortfile)
+
+	var connect, relay, dir string
+	var join, test bool
+
+	flag.StringVar(&connect, "connect", "", "Device ID to which to connect to")
+	flag.BoolVar(&join, "join", false, "Join relay")
+	flag.BoolVar(&test, "test", false, "Generic relay test")
+	flag.StringVar(&relay, "relay", "relay://127.0.0.1:22067", "Relay address")
+	flag.StringVar(&dir, "keys", ".", "Directory where cert.pem and key.pem is stored")
+
+	flag.Parse()
+
+	certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem")
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		log.Fatalln("Failed to load X509 key pair:", err)
+	}
+
+	id := syncthingprotocol.NewDeviceID(cert.Certificate[0])
+	log.Println("ID:", id)
+
+	uri, err := url.Parse(relay)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	stdin := make(chan string)
+
+	go stdinReader(stdin)
+
+	if join {
+		log.Println("Creating client")
+		relay, err := client.NewClient(uri, []tls.Certificate{cert}, nil, 10*time.Second)
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Println("Created client")
+
+		go relay.Serve()
+
+		recv := make(chan protocol.SessionInvitation)
+
+		go func() {
+			log.Println("Starting invitation receiver")
+			for invite := range relay.Invitations() {
+				select {
+				case recv <- invite:
+					log.Println("Received invitation", invite)
+				default:
+					log.Println("Discarding invitation", invite)
+				}
+			}
+		}()
+
+		for {
+			conn, err := client.JoinSession(<-recv)
+			if err != nil {
+				log.Fatalln("Failed to join", err)
+			}
+			log.Println("Joined", conn.RemoteAddr(), conn.LocalAddr())
+			connectToStdio(stdin, conn)
+			log.Println("Finished", conn.RemoteAddr(), conn.LocalAddr())
+		}
+	} else if connect != "" {
+		id, err := syncthingprotocol.DeviceIDFromString(connect)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		invite, err := client.GetInvitationFromRelay(uri, id, []tls.Certificate{cert}, 10*time.Second)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		log.Println("Received invitation", invite)
+		conn, err := client.JoinSession(invite)
+		if err != nil {
+			log.Fatalln("Failed to join", err)
+		}
+		log.Println("Joined", conn.RemoteAddr(), conn.LocalAddr())
+		connectToStdio(stdin, conn)
+		log.Println("Finished", conn.RemoteAddr(), conn.LocalAddr())
+	} else if test {
+		if client.TestRelay(uri, []tls.Certificate{cert}, time.Second, 2*time.Second, 4) {
+			log.Println("OK")
+		} else {
+			log.Println("FAIL")
+		}
+	} else {
+		log.Fatal("Requires either join or connect")
+	}
+}
+
+func stdinReader(c chan<- string) {
+	scanner := bufio.NewScanner(os.Stdin)
+	for scanner.Scan() {
+		c <- scanner.Text()
+		c <- "\n"
+	}
+}
+
+func connectToStdio(stdin <-chan string, conn net.Conn) {
+	go func() {
+
+	}()
+
+	buf := make([]byte, 1024)
+	for {
+		conn.SetReadDeadline(time.Now().Add(time.Millisecond))
+		n, err := conn.Read(buf[0:])
+		if err != nil {
+			nerr, ok := err.(net.Error)
+			if !ok || !nerr.Timeout() {
+				log.Println(err)
+				return
+			}
+		}
+		os.Stdout.Write(buf[:n])
+
+		select {
+		case msg := <-stdin:
+			_, err := conn.Write([]byte(msg))
+			if err != nil {
+				return
+			}
+		default:
+		}
+	}
+}

cmd/strelaysrv/utils.go

@@ -0,0 +1,28 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors.
+
+package main
+
+import (
+	"errors"
+	"net"
+)
+
+func setTCPOptions(conn net.Conn) error {
+	tcpConn, ok := conn.(*net.TCPConn)
+	if !ok {
+		return errors.New("Not a TCP connection")
+	}
+	if err := tcpConn.SetLinger(0); err != nil {
+		return err
+	}
+	if err := tcpConn.SetNoDelay(true); err != nil {
+		return err
+	}
+	if err := tcpConn.SetKeepAlivePeriod(networkTimeout); err != nil {
+		return err
+	}
+	if err := tcpConn.SetKeepAlive(true); err != nil {
+		return err
+	}
+	return nil
+}

cmd/stsigtool/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -24,7 +24,7 @@ func main() {
 	flag.Parse()
 
 	if flag.NArg() < 1 {
-		log.Println(`Usage:
+		log.Print(`Usage:
 	stsigtool <command>
 
 Where command is one of:
@@ -40,6 +40,7 @@ Where command is one of:
 
 	verify <signaturefile> <datafile> <pubkeyfile>
 		- verify a signature, using the specified public key file
+
 `)
 	}
 

cmd/stupgrades/main.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"os"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/upgrade"
+)
+
+const defaultURL = "https://api.github.com/repos/syncthing/syncthing/releases?per_page=25"
+
+func main() {
+	url := flag.String("u", defaultURL, "GitHub releases url")
+	flag.Parse()
+
+	rels := upgrade.FetchLatestReleases(*url, "")
+	if rels == nil {
+		// An error was already logged
+		os.Exit(1)
+	}
+
+	sort.Sort(upgrade.SortByRelease(rels))
+	rels = filterForLatest(rels)
+
+	if err := json.NewEncoder(os.Stdout).Encode(rels); err != nil {
+		os.Exit(1)
+	}
+}
+
+// filterForLatest returns the latest stable and prerelease only. If the
+// stable version is newer (comes first in the list) there is no need to go
+// looking for a prerelease at all.
+func filterForLatest(rels []upgrade.Release) []upgrade.Release {
+	var filtered []upgrade.Release
+	var havePre bool
+	for _, rel := range rels {
+		if !rel.Prerelease {
+			// We found a stable version, we're good now.
+			filtered = append(filtered, rel)
+			break
+		}
+		if rel.Prerelease && !havePre {
+			// We remember the first prerelease we find.
+			filtered = append(filtered, rel)
+			havePre = true
+		}
+	}
+	return filtered
+}

cmd/stvanity/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -130,7 +130,7 @@ func printProgress(prefix string, count *int64) {
 	expectedIterations := float64(int(1) << uint(wantBits))
 	fmt.Printf("Want %d bits for prefix %q, about %.2g certs to test (statistically speaking)\n", wantBits, prefix, expectedIterations)
 
-	for _ = range time.NewTicker(15 * time.Second).C {
+	for range time.NewTicker(15 * time.Second).C {
 		tried := atomic.LoadInt64(count)
 		elapsed := time.Since(started)
 		rate := float64(tried) / elapsed.Seconds()

cmd/stwatchfile/main.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/auditservice.go

@@ -1,69 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"encoding/json"
-	"io"
-
-	"github.com/syncthing/syncthing/lib/events"
-)
-
-// The auditService subscribes to events and writes these in JSON format, one
-// event per line, to the specified writer.
-type auditService struct {
-	w       io.Writer     // audit destination
-	stop    chan struct{} // signals time to stop
-	started chan struct{} // signals startup complete
-	stopped chan struct{} // signals stop complete
-}
-
-func newAuditService(w io.Writer) *auditService {
-	return &auditService{
-		w:       w,
-		stop:    make(chan struct{}),
-		started: make(chan struct{}),
-		stopped: make(chan struct{}),
-	}
-}
-
-// Serve runs the audit service.
-func (s *auditService) Serve() {
-	defer close(s.stopped)
-	sub := events.Default.Subscribe(events.AllEvents)
-	defer events.Default.Unsubscribe(sub)
-	enc := json.NewEncoder(s.w)
-
-	// We're ready to start processing events.
-	close(s.started)
-
-	for {
-		select {
-		case ev := <-sub.C():
-			enc.Encode(ev)
-		case <-s.stop:
-			return
-		}
-	}
-}
-
-// Stop stops the audit service.
-func (s *auditService) Stop() {
-	close(s.stop)
-}
-
-// WaitForStart returns once the audit service is ready to receive events, or
-// immediately if it's already running.
-func (s *auditService) WaitForStart() {
-	<-s.started
-}
-
-// WaitForStop returns once the audit service has stopped.
-// (Needed by the tests.)
-func (s *auditService) WaitForStop() {
-	<-s.stopped
-}

cmd/syncthing/blockprof.go

@@ -2,7 +2,7 @@
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -22,11 +22,15 @@ func init() {
 			panic("Couldn't find block profiler")
 		}
 		l.Debugln("Starting block profiling")
-		go saveBlockingProfiles(profiler)
+		go func() {
+			err := saveBlockingProfiles(profiler) // Only returns on error
+			l.Warnln("Block profiler failed:", err)
+			panic("Block profiler failed")
+		}()
 	}
 }
 
-func saveBlockingProfiles(profiler *pprof.Profile) {
+func saveBlockingProfiles(profiler *pprof.Profile) error {
 	runtime.SetBlockProfileRate(1)
 
 	t0 := time.Now()
@@ -35,16 +39,16 @@ func saveBlockingProfiles(profiler *pprof.Profile) {
 
 		fd, err := os.Create(fmt.Sprintf("block-%05d-%07d.pprof", syscall.Getpid(), startms))
 		if err != nil {
-			panic(err)
+			return err
 		}
 		err = profiler.WriteTo(fd, 0)
 		if err != nil {
-			panic(err)
+			return err
 		}
 		err = fd.Close()
 		if err != nil {
-			panic(err)
+			return err
 		}
-
 	}
+	return nil
 }