build: Dockerfile that expects ready baked binaries

Most notably, it now detects all-lowercase files and returns these
as-is. The tests have been expanded with two cases and are now used
as a benchmark (admittedly a rather trivial one).

name                           old time/op    new time/op    delta
UnicodeLowercaseMaybeChange-8    4.59µs ± 2%    4.57µs ± 1%    ~     (p=0.197 n=10+10)
UnicodeLowercaseNoChange-8       3.26µs ± 1%    3.09µs ± 1%  -5.27%  (p=0.000 n=9+10)

.codecov.yml

@@ -0,0 +1,5 @@
+coverage:
+  range: "40...100"
+
+ignore:
+  - "**.pb.go"

.github/FUNDING.yml

@@ -0,0 +1,11 @@
+github: syncthing
+custom: "https://syncthing.net/donations/"
+
+# patreon: # Replace with a single Patreon username
+# open_collective: # Replace with a single Open Collective username
+# ko_fi: # Replace with a single Ko-fi username
+# tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+# community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+# liberapay: # Replace with a single Liberapay username
+# issuehunt: # Replace with a single IssueHunt username
+# otechie: # Replace with a single Otechie username

.github/ISSUE_TEMPLATE.md

@@ -1,34 +0,0 @@
-### DO NOT REPORT SECURITY ISSUES IN THIS ISSUE TRACKER
-
-Instead, contact security@syncthing.net directly - see
-https://syncthing.net/security.html for more information.
-
-### DO NOT POST SUPPORT REQUESTS OR GENERAL QUESTIONS IN THIS ISSUE TRACKER
-
-Please use the forum at https://forum.syncthing.net/ where a large number of
-helpful people hang out. This issue tracker is for reporting bugs or feature
-requests directly to the developers. Worst case you might get a short
-"that's a bug, please report it on GitHub" response on the forum, in which
-case we thank you for your patience and following our advice. :)
-
-### Please do post actual bug reports and feature requests.
-
-If your issue is a bug report, replace this boilerplate with a description
-of the problem, being sure to include at least:
-
- - what happened,
- - what you expected to happen instead, and
- - any steps to reproduce the problem.
-
-Also fill out the version information below and add log output or
-screenshots as appropriate.
-
-If your issue is a feature request, simply replace this template text in
-its entirety.
-
-### Version Information
-
-Syncthing Version: v0.x.y
-OS Version: Windows 7 / Ubuntu 14.04 / ...
-Browser Version: (if applicable, for GUI issues)
-

.github/ISSUE_TEMPLATE/01-feature.md

@@ -0,0 +1,13 @@
+---
+name: Feature request
+about: If you're just not sure how to do something, see "ask a question".
+labels: enhancement, needs-triage
+---
+
+### Include required information
+
+Please be sure to include at least:
+
+ - what problem your new feature would solve
+ - how or why you think it is generally useful (i.e., not just for you)
+ - what alternatives or workarounds you considered

.github/ISSUE_TEMPLATE/02-bug.md

@@ -0,0 +1,23 @@
+---
+name: Bug report
+about: If you're actually looking for support, see "ask a question".
+labels: bug, needs-triage
+---
+
+### Does your log mention database corruption?
+
+If your Syncthing log reports panics because of database corruption it is
+most likely a fault with your system's storage or memory. Affected log
+entries will contain lines starting with `panic: leveldb`. You will need to
+delete the index database to clear this, by running `syncthing
+-reset-database`.
+
+### Include required information
+
+Please be sure to include at least:
+
+ - which version of Syncthing and what operating system you are using
+ - browser and version, if applicable
+ - what happened,
+ - what you expected to happen instead, and
+ - any steps to reproduce the problem.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a question
+    url: https://forum.syncthing.net/
+    about: Ask questions, get support, and discuss with other community members.
+  - name: Android issues
+    url: https://github.com/syncthing/syncthing-android/issues/
+    about: The Android app has its own issue tracker.

.github/PULL_REQUEST_TEMPLATE.md

@@ -20,13 +20,8 @@ If this is a user visible change (including API and protocol changes), add a lin
 to the corresponding pull request on https://github.com/syncthing/docs or describe
 the documentation changes necessary.
 
-### Authorship
+## Authorship
+
+Your name and email will be added automatically to the AUTHORS file
+based on the commit metadata.
 
-Every author of a code contribution (Go, Javascript, HTML, CSS etc, with the
-possible exception of minor typo corrections and similar) is recorded in the
-AUTHORS and NICKS files and the in-GUI credits. If this is your first
-contribution, a maintainer will add you properly before accepting the
-contribution. You need not do so yourself or worry about the fact that the
-"authors" automated test fails. However, if your name (such as you want it
-presented in the credits) is not visible on your Github profile or in your
-commit messages, please assist by providing it here.

.github/SECURITY.md

@@ -0,0 +1,10 @@
+## Reporting a Vulnerability
+
+If you believe that you've found a Syncthing-related security vulnerability,
+please report it by sending email to the address security@syncthing.net. The
+[PGP key for security@syncthing.net
+(B683AD7B76CAB013)](https://syncthing.net/security-key.txt) can be used to
+send encrypted mail or to verify responses received from that address.
+
+You can read more about Syncthing security at
+https://syncthing.net/security/.

.gitignore

@@ -15,5 +15,6 @@ coverage.xml
 syncthing.sig
 RELEASE
 deb
-lib/auto/gui.files.go
-snapcraft.yaml
+*.bz2
+/repos
+/proto/scripts/protoc-gen-gosyncthing

.golangci.yml

@@ -0,0 +1,26 @@
+linters-settings:
+  maligned:
+    suggest-new: true
+
+linters:
+  enable-all: true
+  disable:
+    - goimports
+    - depguard
+    - lll
+    - gochecknoinits
+    - gochecknoglobals
+    - gofmt
+    - scopelint
+    - gocyclo
+    - funlen
+    - wsl
+    - gocognit
+    - godox
+
+service:
+  golangci-lint-version: 1.21.x
+  prepare:
+    - rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
+    - GO111MODULE=on go mod vendor
+    - go run build.go assets

AUTHORS

@@ -1,41 +1,75 @@
 # This is the official list of Syncthing authors for copyright purposes.
-# The format is:
+#
+# THIS FILE IS MOSTLY AUTO GENERATED. IF YOU'VE MADE A COMMIT TO THE
+# REPOSITORY YOU WILL BE ADDED HERE AUTOMATICALLY WITHOUT THE NEED FOR
+# ANY MANUAL ACTION.
+#
+# That said, you are welcome to correct your name or add a nickname / GitHub
+# user name as appropriate. The format is:
 #
 #    Name Name Name (nickname) <email1@example.com> <email2@example.com>
 #
-# After changing this list, run "go run script/authors.go" to sort and update
-# the GUI HTML.
+# The in-GUI authors list is periodically automatically updated from the
+# contents of this file.
 #
 
 Aaron Bieber (qbit) <qbit@deftly.net>
-Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com>
+Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com> <adam@proactiveservices.co.uk>
 Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
+Alan Pope <alan@popey.com>
+Alberto Donato <albertodonato@users.noreply.github.com>
 Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
+Alex Lindeman <139387+aelindeman@users.noreply.github.com>
+Alex Xu <alex.hello71@gmail.com>
 Alexander Graf (alex2108) <register-github@alex-graf.de>
 Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
+Aman Gupta <aman@tmm1.net>
 Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
+andresvia <andres.via@gmail.com>
 Andrew Dunham (andrew-d) <andrew@du.nham.ca>
-Andrey D (scienmind) <scintertech@cryptolab.net>
+Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
+Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
+André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
+andyleap <andyleap@gmail.com>
+Anjan Momi <anjan@momi.ca>
 Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
 Antony Male (canton7) <antony.male@gmail.com>
+Aranjedeath <Aranjedeath@users.noreply.github.com>
+Arkadiusz Tymiński <gevleeog@gmail.com>
 Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
-Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com>
+Artur Zubilewicz <AkaZecik@users.noreply.github.com>
+Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
+Aurélien Rainone <476650+arl@users.noreply.github.com>
+BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
 Bart De Vries (mogwa1) <devriesb@gmail.com>
 Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
 Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
 Ben Shepherd (benshep) <bjashepherd@gmail.com>
 Ben Sidhom (bsidhom) <bsidhom@gmail.com>
+Benedikt Heine (bebehei) <bebe@bebehei.de>
+Benedikt Morbach <benedikt.morbach@googlemail.com>
+Benno Fünfstück <benno.fuenfstueck@gmail.com>
 Benny Ng (tpng) <benny.tpng@gmail.com>
+boomsquared <54829195+boomsquared@users.noreply.github.com>
+Boqin Qin <bobbqqin@bupt.edu.cn>
+Boris Rybalkin <ribalkin@gmail.com>
 Brandon Philips (philips) <brandon@ifup.org>
 Brendan Long (brendanlong) <self@brendanlong.com>
 Brian R. Becker (brbecker) <brbecker@gmail.com>
 Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
-Carsten Hagemann (Moter8) <moter8@gmail.com>
-Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
+Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
+Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
 Cedric Staniewski (xduugu) <cedric@gmx.ca>
+chenrui <rui@meetup.com>
 Chris Howie (cdhowie) <me@chrishowie.com>
 Chris Joel (cdata) <chris@scriptolo.gy>
+Chris Tonkinson <chris@masterbran.ch>
+chucic <chucic@seznam.cz>
 Colin Kennedy (moshen) <moshen.colin@gmail.com>
+Cromefire_ <tim.l@nghorst.net> <26320625+cromefire@users.noreply.github.com>
+Cyprien Devillez <cypx@users.noreply.github.com>
+Dale Visser <dale.visser@live.com>
+Dan <benda.daniel@gmail.com>
 Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
 Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
 Daniel Martí (mvdan) <mvdan@mvdan.cc>
@@ -43,85 +77,179 @@ Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
 David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
 Denis A. (dva) <denisva@gmail.com>
 Dennis Wilson (snnd) <dw@risu.io>
+dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com> <27856297+dependabot-preview[bot]@users.noreply.github.com>
+dependabot[bot] <dependabot[bot]@users.noreply.github.com>
+derekriemer <derek.riemer@colorado.edu>
+desbma <desbma@users.noreply.github.com>
 Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
+Domenic Horner <domenic@tgxn.net>
 Dominik Heidler (asdil12) <dominik@heidler.eu>
 Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
+Elliot Huffman <thelich2@gmail.com>
 Emil Hessman (ceh) <emil@hessman.se>
 Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
+Evgeny Kuznetsov <evgeny@kuznetsov.md>
 Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
 Felix Ableitner (Nutomic) <me@nutomic.com>
 Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
 Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
 Frank Isemann (fti7) <frank@isemann.name>
+georgespatton <georgespatton@users.noreply.github.com>
+ghjklw <malo@jaffre.info>
 Gilli Sigurdsson (gillisig) <gilli@vx.is>
+Gleb Sinyavskiy <zhulik.gleb@gmail.com>
+Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
+greatroar <61184462+greatroar@users.noreply.github.com>
+Han Boetes <han@boetes.org>
+HansK-p <42314815+HansK-p@users.noreply.github.com>
+Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
 Heiko Zuerker (Smiley73) <heiko@zuerker.org>
+Hugo Locurcio <hugo.locurcio@hugo.pro>
+Iain Barnett <iainspeed@gmail.com>
+Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
+Ilya Brin <464157+ilyabrin@users.noreply.github.com>
+Iskander Sharipov (Alex) <quasilyte@gmail.com>
 Jaakko Hannikainen (jgke) <jgke@jgke.fi>
 Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
+Jack Croft <jccroft1@users.noreply.github.com>
+Jacob <jyundt@gmail.com>
 Jake Peterson (acogdev) <jake@acogdev.com>
 Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
 James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
+janost <janost@tuta.io>
+Jaroslav Lichtblau <svetlemodry@users.noreply.github.com>
 Jaroslav Malec (dzarda) <dzardacz@gmail.com>
+jaseg <githubaccount@jaseg.net>
 Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
+jelle van der Waa <jelle@vdwaa.nl>
 Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
+Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
 Jochen Voss (seehuhn) <voss@seehuhn.de>
+Johan Andersson <j@i19.se>
 Johan Vromans (sciurius) <jvromans@squirrel.nl>
+John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
+Jonas Thelemann <e-mail@jonas-thelemann.de>
+Jonathan <artback@protonmail.com>
+Jonathan Cross <jcross@gmail.com>
 Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
+Jörg Thalheim <Mic92@users.noreply.github.com>
+Jędrzej Kula <kula.jedrek@gmail.com>
+Kalle Laine <pahakalle@protonmail.com>
 Karol Różycki (krozycki) <rozycki.karol@gmail.com>
+Keith Turner <kturner@apache.org>
 Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
 Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
 Kevin Allen (ironmig) <kma1660@gmail.com>
+Kevin Bushiri (keevBush) <keevbush@gmail.com> <36192217+keevBush@users.noreply.github.com>
 Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
+klemens <ka7@github.com>
 Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
 Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
+Laurent Arnoud <laurent@spkdev.net>
 Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
 Leo Arias (elopio) <yo@elopio.net>
 Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
 Lode Hoste (Zillode) <zillode@zillode.be>
 Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
+Lukas Lihotzki <lukas@lihotzki.de>
 Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
 Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
 Marc Pujol (kilburn) <kilburn@la3.org>
 Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
+marco-m <marco.molteni@laposte.net>
+Marcus Legendre <marcus.legendre@gmail.com>
+Mario Majila <mariustshipichik@gmail.com>
 Mark Pulford (mpx) <mark@kyne.com.au>
 Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
+Mateusz Ż <thedead4fun@live.com>
+Matic Potočnik <hairyfotr@gmail.com>
 Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
+Matt Robenolt <matt@ydekproductions.com>
+Matteo Ruina <matteo.ruina@gmail.com>
+Maurizio Tomasi <ziotom78@gmail.com>
 Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
+MaximAL <almaximal@ya.ru>
+Maxime Thirouin <m@moox.io>
 Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
 Michael Ploujnikov (plouj) <ploujj@gmail.com>
+Michael Rienstra <mrienstra@gmail.com>
 Michael Tilli (pyfisch) <pyfisch@gmail.com>
+Mike Boone <mike@boonedocks.net>
+MikeLund <MikeLund@users.noreply.github.com>
+MikolajTwarog <43782609+MikolajTwarog@users.noreply.github.com>
+Mingxuan Lin <gdlmx@users.noreply.github.com>
+mv1005 <49659413+mv1005@users.noreply.github.com>
 Nate Morrison (nrm21) <natemorrison@gmail.com>
 Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
+Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
+Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
+Nicolas Perraut <n.perraut@gmail.com>
 Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
+Nils Jakobi (thunderstorm99) <jakobi.nils@gmail.com>
+NinoM4ster <ninom4ster@gmail.com>
+Nitroretro <43112364+Nitroretro@users.noreply.github.com>
+NoLooseEnds <jon.koslung@gmail.com>
+Oliver Freyermuth <o.freyermuth@googlemail.com>
+otbutz <tbutz@optitool.de>
+Otiel <Otiel@users.noreply.github.com>
+Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
+Pablo <pbaeyens31+github@gmail.com>
 Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
+Paul Brit <paulbrit44@gmail.com>
 Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
+Paweł Rozlach <vespian@users.noreply.github.com>
+perewa <cavalcante.ten@gmail.com>
+Peter Badida <KeyWeeUsr@users.noreply.github.com>
+Peter Dave Hello <hsu@peterdavehello.org>
 Peter Hoeg (peterhoeg) <peter@speartail.com>
+Peter Marquardt (wwwutz) <wwwutz@gmail.com> <wwwutz@googlemail.com>
+Phil Davis <phil.davis@inf.org>
 Philippe Schommers (filoozoom) <philippe@schommers.be>
 Phill Luby (pluby) <phill.luby@newredo.com>
+Pier Paolo Ramon <ramonpierre@gmail.com>
 Piotr Bejda (piobpl) <piotrb10@gmail.com>
+Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
+Rahmi Pruitt <rjpruitt16@gmail.com>
+Richard Hartmann <RichiH@users.noreply.github.com>
 Robert Carosi (nov1n) <robert@carosi.nl>
+Robin Schoonover <robin@cornhooves.org>
 Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
 Ross Smith II (rasa) <ross@smithii.com>
+rubenbe <github-com-00ff86@vandamme.email>
+Ruslan Yevdokymov <38809160+ruslanye@users.noreply.github.com>
 Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
 Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
 Scott Klupfel (kluppy) <kluppy@going2blue.com>
 Sergey Mishin (ralder) <ralder@yandex.ru>
+Shaarad Dalvi <60266155+shaaraddalvi@users.noreply.github.com>
 Simon Frei (imsodin) <freisim93@gmail.com>
+Simon Mwepu <simonmwepu@gmail.com>
+Sly_tom_cat <slytomcat@mail.ru>
 Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
-Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
+Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
 Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
 Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
+Thomas Hipp <thomashipp@gmail.com>
 Tim Abell (timabell) <tim@timwise.co.uk>
 Tim Howes (timhowes) <timhowes@berkeley.edu>
 Tobias Nygren (tnn2) <tnn@nygren.pp.se>
 Tobias Tom (tobiastom) <t.tom@succont.de>
-Tomas Cerveny (kozec) <kozec@kozec.com>
+Tom Jakubowski <tom@crystae.net>
+Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com> <twilczynski@naver.com>
+Tommy Thorn <tommy-github-email@thorn.ws>
 Tully Robinson (tojrobinson) <tully@tojr.org>
 Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
+Tyler Kropp <kropptyler@gmail.com>
 Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
 Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
 Victor Buinsky (buinsky) <vix_booja@tut.by>
 Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
+Vladimir Rusinov <vrusinov@google.com>
+wangguoliang <liangcszzu@163.com>
 William A. Kennington III (wkennington) <william@wkennington.com>
 Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
+xarx00 <xarx00@users.noreply.github.com>
 Xavier O. (damajor) <damajor@gmail.com>
+xjtdy888 (xjtdy888) <xjtdy888@163.com>
 Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
+佛跳墙 <daoquan@qq.com>

CONDUCT.md

@@ -1,92 +1,73 @@
-## Conduct
+# Contributor Covenant Code of Conduct
 
-* We are committed to providing a friendly, safe and welcoming
-  environment for all, regardless of gender, sexual orientation,
-  disability, ethnicity, religion, or similar personal characteristic.
+## Our Pledge
 
-* On IRC, please avoid using overtly sexual nicknames or other nicknames
-  that might detract from a friendly, safe and welcoming environment for
-  all.
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+education, socio-economic status, nationality, personal appearance, race,
+religion, or sexual identity and orientation.
 
-* Please be kind and courteous. There's no need to be mean or rude.
+## Our Standards
 
-* Respect that people have differences of opinion and that every design
-  or implementation choice carries a trade-off and numerous costs. There
-  is seldom a right answer.
+Examples of behavior that contributes to creating a positive environment
+include:
 
-* Please keep unstructured critique to a minimum. If you have solid
-  ideas you want to experiment with, make a fork and see how it works.
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
 
-* We will exclude you from interaction if you insult, demean or harass
-  anyone. That is not welcome behaviour. We interpret the term
-  "harassment" as including the definition in the <a
-  href="http://citizencodeofconduct.org/">Citizen Code of Conduct</a>;
-  if you have any lack of clarity about what might be included in that
-  concept, please read their definition. In particular, we don't
-  tolerate behavior that excludes people in socially marginalized
-  groups.
+Examples of unacceptable behavior by participants include:
 
-* Private harassment is also unacceptable. No matter who you are, if you
-  feel you have been or are being harassed or made uncomfortable by a
-  community member, please contact one of the channel ops or any of the
-  Syncthing core team immediately. Whether you're a regular  contributor
-  or a newcomer, we care about making this community a safe place for
-  you and we've got your back.
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
 
-* Likewise any spamming, trolling, flaming, baiting or other
-  attention-stealing behaviour is not welcome.
+## Our Responsibilities
 
-## Moderation
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
 
-These are the policies for upholding our community's standards of
-conduct in our communication channels, most notably in Syncthing-related
-IRC channels and on the web forum.
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
 
-1. Remarks that violate the Syncthing standards of conduct, including
-   hateful, hurtful, oppressive, or exclusionary remarks, are not
-   allowed. (Cursing is allowed, but never targeting another user, and
-   never in a hateful manner.)
+## Scope
 
-2. Remarks that moderators find inappropriate, whether listed in the
-   code of conduct or not, are also not allowed.
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
 
-3. Moderators will first respond to such remarks with a warning.
+## Enforcement
 
-4. If the warning is unheeded, the user will be "kicked," i.e., kicked
-   out of the communication channel to cool off.
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at security@syncthing.net. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
 
-5. If the user comes back and continues to make trouble, they will be
-   banned, i.e., indefinitely excluded.
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
 
-6. Moderators may choose at their discretion to un-ban the user if it
-   was a first offense and they offer the offended party a genuine
-   apology.
+## Attribution
 
-7. If a moderator bans someone and you think it was unjustified, please
-   take it up with that moderator, or with a different moderator, **in
-   private**. Complaints about bans in-channel are not allowed.
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
 
-8. Moderators are held to a higher standard than other community
-   members. If a moderator creates an inappropriate situation, they
-   should expect less leeway than others.
-
-In the Syncthing community we strive to go the extra step to look out
-for each other. Don't just aim to be technically unimpeachable, try to
-be your best self. In particular, avoid flirting with offensive or
-sensitive issues, particularly if they're off-topic; this all too
-often leads to unnecessary fights, hurt feelings, and damaged trust;
-worse, it can drive people away from the community entirely.
-
-And if someone takes issue with something you said or did, resist the
-urge to be defensive. Just stop doing what it was they complained about
-and apologize. Even if you feel you were misinterpreted or unfairly
-accused, chances are good there was something you could've communicated
-better — remember that it's your responsibility to make your fellow
-community members comfortable. Everyone wants to get along and we are
-all here first and foremost because we want to talk about cool
-technology. You will find that people will be eager to assume good
-intent and forgive as long as you earn their trust.
-
-*Adapted from the [Rust Code of Conduct](https://github.com/rust-lang/rust/wiki/Note-development-policy#conduct)*
-
-*Adapted from the [Node.js Policy on Trolling](http://blog.izs.me/post/30036893703/policy-on-trolling)*
+[homepage]: https://www.contributor-covenant.org

Dockerfile

@@ -1,29 +1,29 @@
-FROM golang:1.9 AS builder
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
 
-WORKDIR /go/src/github.com/syncthing/syncthing
+WORKDIR /src
 COPY . .
 
 ENV CGO_ENABLED=0
 ENV BUILD_HOST=syncthing.net
 ENV BUILD_USER=docker
-RUN rm -f syncthing && go run build.go build syncthing
+RUN rm -f syncthing && go run build.go -no-upgrade build syncthing
 
 FROM alpine
 
-RUN apk add --no-cache ca-certificates
+EXPOSE 8384 22000 21027/udp
 
-COPY --from=builder /go/src/github.com/syncthing/syncthing/syncthing /bin/syncthing
+VOLUME ["/var/syncthing"]
 
-RUN echo 'syncthing:x:1000:1000::/var/syncthing:/sbin/nologin' >> /etc/passwd \
-    && echo 'syncthing:!::0:::::' >> /etc/shadow \
-    && mkdir /var/syncthing \
-    && chown syncthing /var/syncthing
+RUN apk add --no-cache ca-certificates su-exec tzdata
 
-USER syncthing
-ENV STNOUPGRADE=1
+COPY --from=builder /src/syncthing /bin/syncthing
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/syncthing
 
 HEALTHCHECK --interval=1m --timeout=10s \
-  CMD nc -z localhost 8384 || exit 1
-
-ENTRYPOINT ["/bin/syncthing", "-home", "/var/syncthing/config", "-gui-address", "0.0.0.0:8384"]
+  CMD nc -z 127.0.0.1 8384 || exit 1
 
+ENV STGUIADDRESS=0.0.0.0:8384
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]

Dockerfile.builder

@@ -0,0 +1,9 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION
+
+# FPM to build Debian packages
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	locales rubygems ruby-dev build-essential git \
+	&& apt-get clean \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& gem install --no-ri --no-rdoc fpm

Dockerfile.buildx

@@ -0,0 +1,19 @@
+FROM alpine
+ARG TARGETARCH
+
+EXPOSE 8384 22000 21027/udp
+
+VOLUME ["/var/syncthing"]
+
+RUN apk add --no-cache ca-certificates su-exec tzdata
+
+COPY ./syncthing-linux-$TARGETARCH /bin/syncthing
+COPY ./script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/syncthing
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z 127.0.0.1 8384 || exit 1
+
+ENV STGUIADDRESS=0.0.0.0:8384
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]

Dockerfile.stdiscosrv

@@ -0,0 +1,29 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f stdiscosrv && go run build.go -no-upgrade build stdiscosrv
+
+FROM alpine
+
+EXPOSE 19200 8443
+
+VOLUME ["/var/stdiscosrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/stdiscosrv /bin/stdiscosrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/stdiscosrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 8443 || exit 1
+
+WORKDIR /var/stdiscosrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/stdiscosrv"]

Dockerfile.strelaysrv

@@ -0,0 +1,29 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f strelaysrv && go run build.go -no-upgrade build strelaysrv
+
+FROM alpine
+
+EXPOSE 22067 22070
+
+VOLUME ["/var/strelaysrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/strelaysrv /bin/strelaysrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/strelaysrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 22067 || exit 1
+
+WORKDIR /var/strelaysrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaysrv"]

README-Docker.md

@@ -0,0 +1,59 @@
+# Docker Container for Syncthing
+
+Use the Dockerfile in this repo, or pull the `syncthing/syncthing` image
+from Docker Hub.
+
+Use the `/var/syncthing` volume to have the synchronized files available on the
+host. You can add more folders and map them as you prefer.
+
+Note that Syncthing runs as UID 1000 and GID 1000 by default. These may be
+altered with the ``PUID`` and ``PGID`` environment variables.
+
+## Example Usage
+
+```
+$ docker pull syncthing/syncthing
+$ docker run -p 8384:8384 -p 22000:22000 \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+## Discovery
+
+Note that local device discovery will not work with the above command,
+resulting in poor local transfer rates if local device addresses are not
+manually configured.
+
+To allow local discovery, the docker host network can be used instead:
+
+```
+$ docker pull syncthing/syncthing
+$ docker run --network=host \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+Be aware that syncthing alone is now in control of what interfaces and ports it
+listens on. You can edit the syncthing configuration to change the defaults if
+there are conflicts.
+
+## GUI Security
+
+By default Syncthing inside the Docker image listens on 0.0.0.0:8384 to
+allow GUI connections via the Docker proxy. This is set by the
+`STGUIADDRESS` environment variable in the Dockerfile, as it differs from
+what Syncthing would otherwise use by default. This means you should set up
+authentication in the GUI, like for any other externally reachable Syncthing
+instance. If you do not require the GUI, or you use host networking, you can
+unset the `STGUIADDRESS` variable to have Syncthing fall back to listening
+on 127.0.0.1:
+
+```
+$ docker pull syncthing/syncthing
+$ docker run -e STGUIADDRESS= \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+With the environment variable unset Syncthing will follow what is set in the
+configuration file / GUI settings dialog.

README.md

@@ -2,7 +2,6 @@
 
 ---
 
-[![Latest Downloads](https://img.shields.io/badge/latest-downloads-brightgreen.svg?style=flat-square)](https://build.syncthing.net/latest/)
 [![Latest Linux & Cross Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildLinuxCross.svg?style=flat-square&label=linux+%26+cross+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildLinuxCross&guest=1)
 [![Latest Windows Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildWindows.svg?style=flat-square&label=windows+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildWindows&guest=1)
 [![Latest Mac Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildMac.svg?style=flat-square&label=mac+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildMac&guest=1)
@@ -63,6 +62,10 @@ There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory][3]. There are also several [GUI
 implementations][11] for Windows, Mac and Linux.
 
+## Docker
+
+To run Syncthing in Docker, see [the Docker README][16].
+
 ## Vote on features/bugs
 
 We'd like to encourage you to [vote][12] on issues that matter to you.
@@ -98,17 +101,18 @@ All code is licensed under the [MPLv2 License][7].
 
 [1]: https://docs.syncthing.net/specs/bep-v1.html
 [2]: https://docs.syncthing.net/intro/getting-started.html
-[3]: https://github.com/syncthing/syncthing/blob/master/etc
+[3]: https://github.com/syncthing/syncthing/blob/main/etc
 [4]: https://www.freenode.net/
 [5]: https://docs.syncthing.net/dev/building.html
 [6]: https://docs.syncthing.net/
-[7]: https://github.com/syncthing/syncthing/blob/master/LICENSE
+[7]: https://github.com/syncthing/syncthing/blob/main/LICENSE
 [8]: https://forum.syncthing.net/
 [9]: https://kiwiirc.com/client/irc.freenode.net/#syncthing
 [10]: https://github.com/syncthing/syncthing/issues
 [11]: https://docs.syncthing.net/users/contrib.html#gui-wrappers
 [12]: https://www.bountysource.com/teams/syncthing/issues
-[13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
+[13]: https://github.com/syncthing/syncthing/blob/main/GOALS.md
 [14]: assets/logo-text-128.png
 [15]: https://syncthing.net/
+[16]: https://github.com/syncthing/syncthing/blob/main/README-Docker.md
 

build.ps1

@@ -0,0 +1,20 @@
+function build {
+    go run build.go @args
+}
+
+$cmd, $rest = $args
+switch ($cmd) {
+    "test" {
+        $env:LOGGER_DISCARD=1
+        build test
+    }
+
+    "bench" {
+        $env:LOGGER_DISCARD=1
+        build bench
+    }
+
+    default {
+        build @rest
+    }
+}

build.sh

@@ -2,8 +2,6 @@
 set -euo pipefail
 IFS=$'\n\t'
 
-STTRACE=${STTRACE:-}
-
 script() {
 	name="$1"
 	shift
@@ -15,88 +13,23 @@ build() {
 }
 
 case "${1:-default}" in
-	default)
-		build
-		;;
-
-	clean)
-		build "$@"
-		;;
-
-	tar)
-		build "$@"
-		;;
-
-	assets)
-		build "$@"
-		;;
-
-	xdr)
-		build "$@"
-		;;
-
-	translate)
-		build "$@"
-		;;
-
-	deb)
-		build "$@"
-		;;
-
-	setup)
-		build "$@"
-		;;
-
 	test)
 		LOGGER_DISCARD=1 build test
 		;;
 
 	bench)
-		LOGGER_DISCARD=1 build bench | script benchfilter
+		LOGGER_DISCARD=1 build bench
 		;;
 
 	prerelease)
-		go run script/authors.go
+		script authors
 		build transifex
 		pushd man ; ./refresh.sh ; popd
-		git add -A gui man
-		git commit -m 'gui, man: Update docs & translations'
-		;;
-
-	noupgrade)
-		build -no-upgrade tar
-		;;
-
-	all)
-		platforms=(
-			darwin-amd64 dragonfly-amd64 freebsd-amd64 linux-amd64 netbsd-amd64 openbsd-amd64 solaris-amd64 windows-amd64
-			freebsd-386 linux-386 netbsd-386 openbsd-386 windows-386
-			linux-arm linux-arm64 linux-ppc64 linux-ppc64le
-		)
-
-		for plat in "${platforms[@]}"; do
-			echo Building "$plat"
-
-			goos="${plat%-*}"
-			goarch="${plat#*-}"
-			dist="tar"
-
-			if [[ $goos == "windows" ]]; then
-				dist="zip"
-			fi
-
-			build -goos "$goos" -goarch "$goarch" "$dist"
-			echo
-		done
-		;;
-
-	test-xunit)
-
-		(GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -v -race ./lib/... ./cmd/... || true) > tests.out
-		go2xunit -output tests.xml -fail < tests.out
+		git add -A gui man AUTHORS
+		git commit -m 'gui, man, authors: Update docs, translations, and contributors'
 		;;
 
 	*)
-		echo "Unknown build command $1"
+		build "$@"
 		;;
 esac

cmd/stbench/main.go

@@ -1,143 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// This doesn't build on Windows due to the Rusage stuff.
-
-// +build !windows
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"runtime"
-	"syscall"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/rc"
-)
-
-var homeDir = "h1"
-var syncthingBin = "./bin/syncthing"
-var test = "scan"
-
-func main() {
-	flag.StringVar(&homeDir, "home", homeDir, "Home directory location")
-	flag.StringVar(&syncthingBin, "bin", syncthingBin, "Binary location")
-	flag.StringVar(&test, "test", test, "Test to run")
-	flag.Parse()
-
-	switch test {
-	case "scan":
-		// scan measures the resource usage required to perform the initial
-		// scan, without cleaning away the database first.
-		testScan()
-	}
-}
-
-// testScan starts a process and reports on the resource usage required to
-// perform the initial scan.
-func testScan() {
-	log.Println("Starting...")
-	p := rc.NewProcess("127.0.0.1:8081")
-	if err := p.Start(syncthingBin, "-home", homeDir, "-no-browser"); err != nil {
-		log.Println(err)
-		return
-	}
-	defer p.Stop()
-
-	wallTime := awaitScanComplete(p)
-
-	report(p, wallTime)
-}
-
-// awaitScanComplete waits for a folder to transition idle->scanning and
-// then scanning->idle and returns the time taken for the scan.
-func awaitScanComplete(p *rc.Process) time.Duration {
-	log.Println("Awaiting scan completion...")
-	var t0, t1 time.Time
-	lastEvent := 0
-loop:
-	for {
-		evs, err := p.Events(lastEvent)
-		if err != nil {
-			continue
-		}
-
-		for _, ev := range evs {
-			if ev.Type == "StateChanged" {
-				data := ev.Data.(map[string]interface{})
-				log.Println(ev)
-
-				if data["to"].(string) == "scanning" {
-					t0 = ev.Time
-					continue
-				}
-
-				if !t0.IsZero() && data["to"].(string) == "idle" {
-					t1 = ev.Time
-					break loop
-				}
-			}
-			lastEvent = ev.ID
-		}
-
-		time.Sleep(250 * time.Millisecond)
-	}
-
-	return t1.Sub(t0)
-}
-
-// report stops the given process and reports on its resource usage in two
-// ways: human readable to stderr, and CSV to stdout.
-func report(p *rc.Process, wallTime time.Duration) {
-	sv, err := p.SystemVersion()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	ss, err := p.SystemStatus()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	proc, err := p.Stop()
-	if err != nil {
-		return
-	}
-
-	rusage, ok := proc.SysUsage().(*syscall.Rusage)
-	if !ok {
-		return
-	}
-
-	log.Println("Version:", sv.Version)
-	log.Println("Alloc:", ss.Alloc/1024, "KiB")
-	log.Println("Sys:", ss.Sys/1024, "KiB")
-	log.Println("Goroutines:", ss.Goroutines)
-	log.Println("Wall time:", wallTime)
-	log.Println("Utime:", time.Duration(rusage.Utime.Nano()))
-	log.Println("Stime:", time.Duration(rusage.Stime.Nano()))
-	if runtime.GOOS == "darwin" {
-		// Darwin reports in bytes, Linux seems to report in KiB even
-		// though the manpage says otherwise.
-		rusage.Maxrss /= 1024
-	}
-	log.Println("MaxRSS:", rusage.Maxrss, "KiB")
-
-	fmt.Printf("%s,%d,%d,%d,%.02f,%.02f,%.02f,%d\n",
-		sv.Version,
-		ss.Alloc/1024,
-		ss.Sys/1024,
-		ss.Goroutines,
-		wallTime.Seconds(),
-		time.Duration(rusage.Utime.Nano()).Seconds(),
-		time.Duration(rusage.Stime.Nano()).Seconds(),
-		rusage.Maxrss)
-}

cmd/stcli/LICENSE

@@ -1,19 +0,0 @@
-Copyright (C) 2014 Audrius Butkevičius
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-- The above copyright notice and this permission notice shall be included in
-  all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

cmd/stcli/client.go

@@ -1,115 +1,96 @@
-// Copyright (C) 2014 Audrius Butkevičius
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
 	"net/http"
 	"strings"
 
-	"github.com/AudriusButkevicius/cli"
+	"github.com/syncthing/syncthing/lib/config"
 )
 
 type APIClient struct {
-	httpClient http.Client
-	endpoint   string
-	apikey     string
-	username   string
-	password   string
-	id         string
-	csrf       string
+	http.Client
+	cfg    config.GUIConfiguration
+	apikey string
 }
 
-var instance *APIClient
-
-func getClient(c *cli.Context) *APIClient {
-	if instance != nil {
-		return instance
-	}
-	endpoint := c.GlobalString("endpoint")
-	if !strings.HasPrefix(endpoint, "http") {
-		endpoint = "http://" + endpoint
-	}
+func getClient(cfg config.GUIConfiguration) *APIClient {
 	httpClient := http.Client{
 		Transport: &http.Transport{
 			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: c.GlobalBool("insecure"),
+				InsecureSkipVerify: true,
+			},
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial(cfg.Network(), cfg.Address())
 			},
 		},
 	}
-	client := APIClient{
-		httpClient: httpClient,
-		endpoint:   endpoint,
-		apikey:     c.GlobalString("apikey"),
-		username:   c.GlobalString("username"),
-		password:   c.GlobalString("password"),
+	return &APIClient{
+		Client: httpClient,
+		cfg:    cfg,
+		apikey: cfg.APIKey,
 	}
-
-	if client.apikey == "" {
-		request, err := http.NewRequest("GET", client.endpoint, nil)
-		die(err)
-		response := client.handleRequest(request)
-		client.id = response.Header.Get("X-Syncthing-ID")
-		if client.id == "" {
-			die("Failed to get device ID")
-		}
-		for _, item := range response.Cookies() {
-			if item.Name == "CSRF-Token-"+client.id[:5] {
-				client.csrf = item.Value
-				goto csrffound
-			}
-		}
-		die("Failed to get CSRF token")
-	csrffound:
-	}
-	instance = &client
-	return &client
 }
 
-func (client *APIClient) handleRequest(request *http.Request) *http.Response {
-	if client.apikey != "" {
-		request.Header.Set("X-API-Key", client.apikey)
+func (c *APIClient) Endpoint() string {
+	if c.cfg.Network() == "unix" {
+		return "http://unix/"
 	}
-	if client.username != "" || client.password != "" {
-		request.SetBasicAuth(client.username, client.password)
-	}
-	if client.csrf != "" {
-		request.Header.Set("X-CSRF-Token-"+client.id[:5], client.csrf)
+	url := c.cfg.URL()
+	if !strings.HasSuffix(url, "/") {
+		url += "/"
 	}
+	return url
+}
 
-	response, err := client.httpClient.Do(request)
-	die(err)
+func (c *APIClient) Do(req *http.Request) (*http.Response, error) {
+	req.Header.Set("X-API-Key", c.apikey)
+	resp, err := c.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	return resp, checkResponse(resp)
+}
 
+func (c *APIClient) Get(url string) (*http.Response, error) {
+	request, err := http.NewRequest("GET", c.Endpoint()+"rest/"+url, nil)
+	if err != nil {
+		return nil, err
+	}
+	return c.Do(request)
+}
+
+func (c *APIClient) Post(url, body string) (*http.Response, error) {
+	request, err := http.NewRequest("POST", c.Endpoint()+"rest/"+url, bytes.NewBufferString(body))
+	if err != nil {
+		return nil, err
+	}
+	return c.Do(request)
+}
+
+func checkResponse(response *http.Response) error {
 	if response.StatusCode == 404 {
-		die("Invalid endpoint or API call")
-	} else if response.StatusCode == 401 {
-		die("Invalid username or password")
+		return errors.New("invalid endpoint or API call")
 	} else if response.StatusCode == 403 {
-		if client.apikey == "" {
-			die("Invalid CSRF token")
-		}
-		die("Invalid API key")
+		return errors.New("invalid API key")
 	} else if response.StatusCode != 200 {
-		body := strings.TrimSpace(string(responseToBArray(response)))
-		if body != "" {
-			die(body)
+		data, err := responseToBArray(response)
+		if err != nil {
+			return err
 		}
-		die("Unknown HTTP status returned: " + response.Status)
+		body := strings.TrimSpace(string(data))
+		return fmt.Errorf("unexpected HTTP status returned: %s\n%s", response.Status, body)
 	}
-	return response
-}
-
-func httpGet(c *cli.Context, url string) *http.Response {
-	client := getClient(c)
-	request, err := http.NewRequest("GET", client.endpoint+"/rest/"+url, nil)
-	die(err)
-	return client.handleRequest(request)
-}
-
-func httpPost(c *cli.Context, url string, body string) *http.Response {
-	client := getClient(c)
-	request, err := http.NewRequest("POST", client.endpoint+"/rest/"+url, bytes.NewBufferString(body))
-	die(err)
-	return client.handleRequest(request)
+	return nil
 }

cmd/stcli/cmd_devices.go

@@ -1,188 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-	"github.com/syncthing/syncthing/lib/config"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "devices",
-		HideHelp: true,
-		Usage:    "Device command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "list",
-				Usage:    "List registered devices",
-				Requires: &cli.Requires{},
-				Action:   devicesList,
-			},
-			{
-				Name:     "add",
-				Usage:    "Add a new device",
-				Requires: &cli.Requires{"device id", "device name?"},
-				Action:   devicesAdd,
-			},
-			{
-				Name:     "remove",
-				Usage:    "Remove an existing device",
-				Requires: &cli.Requires{"device id"},
-				Action:   devicesRemove,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a property of a device",
-				Requires: &cli.Requires{"device id", "property"},
-				Action:   devicesGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a property of a device",
-				Requires: &cli.Requires{"device id", "property", "value..."},
-				Action:   devicesSet,
-			},
-		},
-	})
-}
-
-func devicesList(c *cli.Context) {
-	cfg := getConfig(c)
-	first := true
-	writer := newTableWriter()
-	for _, device := range cfg.Devices {
-		if !first {
-			fmt.Fprintln(writer)
-		}
-		fmt.Fprintln(writer, "ID:\t", device.DeviceID, "\t")
-		fmt.Fprintln(writer, "Name:\t", device.Name, "\t(name)")
-		fmt.Fprintln(writer, "Address:\t", strings.Join(device.Addresses, " "), "\t(address)")
-		fmt.Fprintln(writer, "Compression:\t", device.Compression, "\t(compression)")
-		fmt.Fprintln(writer, "Certificate name:\t", device.CertName, "\t(certname)")
-		fmt.Fprintln(writer, "Introducer:\t", device.Introducer, "\t(introducer)")
-		first = false
-	}
-	writer.Flush()
-}
-
-func devicesAdd(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-
-	newDevice := config.DeviceConfiguration{
-		DeviceID:  id,
-		Name:      nid,
-		Addresses: []string{"dynamic"},
-	}
-
-	if len(c.Args()) > 1 {
-		newDevice.Name = c.Args()[1]
-	}
-
-	if len(c.Args()) > 2 {
-		addresses := c.Args()[2:]
-		for _, item := range addresses {
-			if item == "dynamic" {
-				continue
-			}
-			validAddress(item)
-		}
-		newDevice.Addresses = addresses
-	}
-
-	cfg := getConfig(c)
-	for _, device := range cfg.Devices {
-		if device.DeviceID == id {
-			die("Device " + nid + " already exists")
-		}
-	}
-	cfg.Devices = append(cfg.Devices, newDevice)
-	setConfig(c, cfg)
-}
-
-func devicesRemove(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-	if nid == getMyID(c) {
-		die("Cannot remove yourself")
-	}
-	cfg := getConfig(c)
-	for i, device := range cfg.Devices {
-		if device.DeviceID == id {
-			last := len(cfg.Devices) - 1
-			cfg.Devices[i] = cfg.Devices[last]
-			cfg.Devices = cfg.Devices[:last]
-			setConfig(c, cfg)
-			return
-		}
-	}
-	die("Device " + nid + " not found")
-}
-
-func devicesGet(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-	arg := c.Args()[1]
-	cfg := getConfig(c)
-	for _, device := range cfg.Devices {
-		if device.DeviceID != id {
-			continue
-		}
-		switch strings.ToLower(arg) {
-		case "name":
-			fmt.Println(device.Name)
-		case "address":
-			fmt.Println(strings.Join(device.Addresses, "\n"))
-		case "compression":
-			fmt.Println(device.Compression.String())
-		case "certname":
-			fmt.Println(device.CertName)
-		case "introducer":
-			fmt.Println(device.Introducer)
-		default:
-			die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
-		}
-		return
-	}
-	die("Device " + nid + " not found")
-}
-
-func devicesSet(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-	arg := c.Args()[1]
-	config := getConfig(c)
-	for i, device := range config.Devices {
-		if device.DeviceID != id {
-			continue
-		}
-		switch strings.ToLower(arg) {
-		case "name":
-			config.Devices[i].Name = strings.Join(c.Args()[2:], " ")
-		case "address":
-			for _, item := range c.Args()[2:] {
-				if item == "dynamic" {
-					continue
-				}
-				validAddress(item)
-			}
-			config.Devices[i].Addresses = c.Args()[2:]
-		case "compression":
-			err := config.Devices[i].Compression.UnmarshalText([]byte(c.Args()[2]))
-			die(err)
-		case "certname":
-			config.Devices[i].CertName = strings.Join(c.Args()[2:], " ")
-		case "introducer":
-			config.Devices[i].Introducer = parseBool(c.Args()[2])
-		default:
-			die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
-		}
-		setConfig(c, config)
-		return
-	}
-	die("Device " + nid + " not found")
-}

cmd/stcli/cmd_errors.go

@@ -1,67 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "errors",
-		HideHelp: true,
-		Usage:    "Error command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "show",
-				Usage:    "Show pending errors",
-				Requires: &cli.Requires{},
-				Action:   errorsShow,
-			},
-			{
-				Name:     "push",
-				Usage:    "Push an error to active clients",
-				Requires: &cli.Requires{"error message..."},
-				Action:   errorsPush,
-			},
-			{
-				Name:     "clear",
-				Usage:    "Clear pending errors",
-				Requires: &cli.Requires{},
-				Action:   wrappedHTTPPost("system/error/clear"),
-			},
-		},
-	})
-}
-
-func errorsShow(c *cli.Context) {
-	response := httpGet(c, "system/error")
-	var data map[string][]map[string]interface{}
-	json.Unmarshal(responseToBArray(response), &data)
-	writer := newTableWriter()
-	for _, item := range data["errors"] {
-		time := item["time"].(string)[:19]
-		time = strings.Replace(time, "T", " ", 1)
-		err := item["error"].(string)
-		err = strings.TrimSpace(err)
-		fmt.Fprintln(writer, time+":\t"+err)
-	}
-	writer.Flush()
-}
-
-func errorsPush(c *cli.Context) {
-	err := strings.Join(c.Args(), " ")
-	response := httpPost(c, "system/error", strings.TrimSpace(err))
-	if response.StatusCode != 200 {
-		err = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
-		body := string(responseToBArray(response))
-		if body != "" {
-			err += "\nBody: " + body
-		}
-		die(err)
-	}
-}

cmd/stcli/cmd_folders.go

@@ -1,361 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"path/filepath"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/fs"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "folders",
-		HideHelp: true,
-		Usage:    "Folder command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "list",
-				Usage:    "List available folders",
-				Requires: &cli.Requires{},
-				Action:   foldersList,
-			},
-			{
-				Name:     "add",
-				Usage:    "Add a new folder",
-				Requires: &cli.Requires{"folder id", "directory"},
-				Action:   foldersAdd,
-			},
-			{
-				Name:     "remove",
-				Usage:    "Remove an existing folder",
-				Requires: &cli.Requires{"folder id"},
-				Action:   foldersRemove,
-			},
-			{
-				Name:     "override",
-				Usage:    "Override changes from other nodes for a master folder",
-				Requires: &cli.Requires{"folder id"},
-				Action:   foldersOverride,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a property of a folder",
-				Requires: &cli.Requires{"folder id", "property"},
-				Action:   foldersGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a property of a folder",
-				Requires: &cli.Requires{"folder id", "property", "value..."},
-				Action:   foldersSet,
-			},
-			{
-				Name:     "unset",
-				Usage:    "Unset a property of a folder",
-				Requires: &cli.Requires{"folder id", "property"},
-				Action:   foldersUnset,
-			},
-			{
-				Name:     "devices",
-				Usage:    "Folder devices command group",
-				HideHelp: true,
-				Subcommands: []cli.Command{
-					{
-						Name:     "list",
-						Usage:    "List of devices which the folder is shared with",
-						Requires: &cli.Requires{"folder id"},
-						Action:   foldersDevicesList,
-					},
-					{
-						Name:     "add",
-						Usage:    "Share a folder with a device",
-						Requires: &cli.Requires{"folder id", "device id"},
-						Action:   foldersDevicesAdd,
-					},
-					{
-						Name:     "remove",
-						Usage:    "Unshare a folder with a device",
-						Requires: &cli.Requires{"folder id", "device id"},
-						Action:   foldersDevicesRemove,
-					},
-					{
-						Name:     "clear",
-						Usage:    "Unshare a folder with all devices",
-						Requires: &cli.Requires{"folder id"},
-						Action:   foldersDevicesClear,
-					},
-				},
-			},
-		},
-	})
-}
-
-func foldersList(c *cli.Context) {
-	cfg := getConfig(c)
-	first := true
-	writer := newTableWriter()
-	for _, folder := range cfg.Folders {
-		if !first {
-			fmt.Fprintln(writer)
-		}
-		fs := folder.Filesystem()
-		fmt.Fprintln(writer, "ID:\t", folder.ID, "\t")
-		fmt.Fprintln(writer, "Path:\t", fs.URI(), "\t(directory)")
-		fmt.Fprintln(writer, "Path type:\t", fs.Type(), "\t(directory-type)")
-		fmt.Fprintln(writer, "Folder type:\t", folder.Type, "\t(type)")
-		fmt.Fprintln(writer, "Ignore permissions:\t", folder.IgnorePerms, "\t(permissions)")
-		fmt.Fprintln(writer, "Rescan interval in seconds:\t", folder.RescanIntervalS, "\t(rescan)")
-
-		if folder.Versioning.Type != "" {
-			fmt.Fprintln(writer, "Versioning:\t", folder.Versioning.Type, "\t(versioning)")
-			for key, value := range folder.Versioning.Params {
-				fmt.Fprintf(writer, "Versioning %s:\t %s \t(versioning-%s)\n", key, value, key)
-			}
-		}
-		first = false
-	}
-	writer.Flush()
-}
-
-func foldersAdd(c *cli.Context) {
-	cfg := getConfig(c)
-	abs, err := filepath.Abs(c.Args()[1])
-	die(err)
-	folder := config.FolderConfiguration{
-		ID:             c.Args()[0],
-		Path:           filepath.Clean(abs),
-		FilesystemType: fs.FilesystemTypeBasic,
-	}
-	cfg.Folders = append(cfg.Folders, folder)
-	setConfig(c, cfg)
-}
-
-func foldersRemove(c *cli.Context) {
-	cfg := getConfig(c)
-	rid := c.Args()[0]
-	for i, folder := range cfg.Folders {
-		if folder.ID == rid {
-			last := len(cfg.Folders) - 1
-			cfg.Folders[i] = cfg.Folders[last]
-			cfg.Folders = cfg.Folders[:last]
-			setConfig(c, cfg)
-			return
-		}
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersOverride(c *cli.Context) {
-	cfg := getConfig(c)
-	rid := c.Args()[0]
-	for _, folder := range cfg.Folders {
-		if folder.ID == rid && folder.Type == config.FolderTypeSendOnly {
-			response := httpPost(c, "db/override", "")
-			if response.StatusCode != 200 {
-				err := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
-				body := string(responseToBArray(response))
-				if body != "" {
-					err += "\nBody: " + body
-				}
-				die(err)
-			}
-			return
-		}
-	}
-	die("Folder " + rid + " not found or folder not master")
-}
-
-func foldersGet(c *cli.Context) {
-	cfg := getConfig(c)
-	rid := c.Args()[0]
-	arg := strings.ToLower(c.Args()[1])
-	for _, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		if strings.HasPrefix(arg, "versioning-") {
-			arg = arg[11:]
-			value, ok := folder.Versioning.Params[arg]
-			if ok {
-				fmt.Println(value)
-				return
-			}
-			die("Versioning property " + c.Args()[1][11:] + " not found")
-		}
-		switch arg {
-		case "directory":
-			fmt.Println(folder.Filesystem().URI())
-		case "directory-type":
-			fmt.Println(folder.Filesystem().Type())
-		case "type":
-			fmt.Println(folder.Type)
-		case "permissions":
-			fmt.Println(folder.IgnorePerms)
-		case "rescan":
-			fmt.Println(folder.RescanIntervalS)
-		case "versioning":
-			if folder.Versioning.Type != "" {
-				fmt.Println(folder.Versioning.Type)
-			}
-		default:
-			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, directory-type, type, permissions, versioning, versioning-<key>")
-		}
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersSet(c *cli.Context) {
-	rid := c.Args()[0]
-	arg := strings.ToLower(c.Args()[1])
-	val := strings.Join(c.Args()[2:], " ")
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		if strings.HasPrefix(arg, "versioning-") {
-			cfg.Folders[i].Versioning.Params[arg[11:]] = val
-			setConfig(c, cfg)
-			return
-		}
-		switch arg {
-		case "directory":
-			cfg.Folders[i].Path = val
-		case "directory-type":
-			var fsType fs.FilesystemType
-			fsType.UnmarshalText([]byte(val))
-			cfg.Folders[i].FilesystemType = fsType
-		case "type":
-			var t config.FolderType
-			if err := t.UnmarshalText([]byte(val)); err != nil {
-				die("Invalid folder type: " + err.Error())
-			}
-			cfg.Folders[i].Type = t
-		case "permissions":
-			cfg.Folders[i].IgnorePerms = parseBool(val)
-		case "rescan":
-			cfg.Folders[i].RescanIntervalS = parseInt(val)
-		case "versioning":
-			cfg.Folders[i].Versioning.Type = val
-		default:
-			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, master, permissions, versioning, versioning-<key>")
-		}
-		setConfig(c, cfg)
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersUnset(c *cli.Context) {
-	rid := c.Args()[0]
-	arg := strings.ToLower(c.Args()[1])
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		if strings.HasPrefix(arg, "versioning-") {
-			arg = arg[11:]
-			if _, ok := folder.Versioning.Params[arg]; ok {
-				delete(cfg.Folders[i].Versioning.Params, arg)
-				setConfig(c, cfg)
-				return
-			}
-			die("Versioning property " + c.Args()[1][11:] + " not found")
-		}
-		switch arg {
-		case "versioning":
-			cfg.Folders[i].Versioning.Type = ""
-			cfg.Folders[i].Versioning.Params = make(map[string]string)
-		default:
-			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: versioning, versioning-<key>")
-		}
-		setConfig(c, cfg)
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesList(c *cli.Context) {
-	rid := c.Args()[0]
-	cfg := getConfig(c)
-	for _, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		for _, device := range folder.Devices {
-			fmt.Println(device.DeviceID)
-		}
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesAdd(c *cli.Context) {
-	rid := c.Args()[0]
-	nid := parseDeviceID(c.Args()[1])
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		for _, device := range folder.Devices {
-			if device.DeviceID == nid {
-				die("Device " + c.Args()[1] + " is already part of this folder")
-			}
-		}
-		for _, device := range cfg.Devices {
-			if device.DeviceID == nid {
-				cfg.Folders[i].Devices = append(folder.Devices, config.FolderDeviceConfiguration{
-					DeviceID: device.DeviceID,
-				})
-				setConfig(c, cfg)
-				return
-			}
-		}
-		die("Device " + c.Args()[1] + " not found in device list")
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesRemove(c *cli.Context) {
-	rid := c.Args()[0]
-	nid := parseDeviceID(c.Args()[1])
-	cfg := getConfig(c)
-	for ri, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		for ni, device := range folder.Devices {
-			if device.DeviceID == nid {
-				last := len(folder.Devices) - 1
-				cfg.Folders[ri].Devices[ni] = folder.Devices[last]
-				cfg.Folders[ri].Devices = cfg.Folders[ri].Devices[:last]
-				setConfig(c, cfg)
-				return
-			}
-		}
-		die("Device " + c.Args()[1] + " not found")
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesClear(c *cli.Context) {
-	rid := c.Args()[0]
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		cfg.Folders[i].Devices = []config.FolderDeviceConfiguration{}
-		setConfig(c, cfg)
-		return
-	}
-	die("Folder " + rid + " not found")
-}

cmd/stcli/cmd_general.go

@@ -1,78 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, []cli.Command{
-		{
-			Name:     "id",
-			Usage:    "Get ID of the Syncthing client",
-			Requires: &cli.Requires{},
-			Action:   generalID,
-		},
-		{
-			Name:     "status",
-			Usage:    "Configuration status, whether or not a restart is required for changes to take effect",
-			Requires: &cli.Requires{},
-			Action:   generalStatus,
-		},
-		{
-			Name:     "restart",
-			Usage:    "Restart syncthing",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/restart"),
-		},
-		{
-			Name:     "shutdown",
-			Usage:    "Shutdown syncthing",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/shutdown"),
-		},
-		{
-			Name:     "reset",
-			Usage:    "Reset syncthing deleting all folders and devices",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/reset"),
-		},
-		{
-			Name:     "upgrade",
-			Usage:    "Upgrade syncthing (if a newer version is available)",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/upgrade"),
-		},
-		{
-			Name:     "version",
-			Usage:    "Syncthing client version",
-			Requires: &cli.Requires{},
-			Action:   generalVersion,
-		},
-	}...)
-}
-
-func generalID(c *cli.Context) {
-	fmt.Println(getMyID(c))
-}
-
-func generalStatus(c *cli.Context) {
-	response := httpGet(c, "system/config/insync")
-	var status struct{ ConfigInSync bool }
-	json.Unmarshal(responseToBArray(response), &status)
-	if !status.ConfigInSync {
-		die("Config out of sync")
-	}
-	fmt.Println("Config in sync")
-}
-
-func generalVersion(c *cli.Context) {
-	response := httpGet(c, "system/version")
-	version := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &version)
-	prettyPrintJSON(version)
-}

cmd/stcli/cmd_gui.go

@@ -1,127 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "gui",
-		HideHelp: true,
-		Usage:    "GUI command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "dump",
-				Usage:    "Show all GUI configuration settings",
-				Requires: &cli.Requires{},
-				Action:   guiDump,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a GUI configuration setting",
-				Requires: &cli.Requires{"setting"},
-				Action:   guiGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a GUI configuration setting",
-				Requires: &cli.Requires{"setting", "value"},
-				Action:   guiSet,
-			},
-			{
-				Name:     "unset",
-				Usage:    "Unset a GUI configuration setting",
-				Requires: &cli.Requires{"setting"},
-				Action:   guiUnset,
-			},
-		},
-	})
-}
-
-func guiDump(c *cli.Context) {
-	cfg := getConfig(c).GUI
-	writer := newTableWriter()
-	fmt.Fprintln(writer, "Enabled:\t", cfg.Enabled, "\t(enabled)")
-	fmt.Fprintln(writer, "Use HTTPS:\t", cfg.UseTLS(), "\t(tls)")
-	fmt.Fprintln(writer, "Listen Addresses:\t", cfg.Address(), "\t(address)")
-	if cfg.User != "" {
-		fmt.Fprintln(writer, "Authentication User:\t", cfg.User, "\t(username)")
-		fmt.Fprintln(writer, "Authentication Password:\t", cfg.Password, "\t(password)")
-	}
-	if cfg.APIKey != "" {
-		fmt.Fprintln(writer, "API Key:\t", cfg.APIKey, "\t(apikey)")
-	}
-	writer.Flush()
-}
-
-func guiGet(c *cli.Context) {
-	cfg := getConfig(c).GUI
-	arg := c.Args()[0]
-	switch strings.ToLower(arg) {
-	case "enabled":
-		fmt.Println(cfg.Enabled)
-	case "tls":
-		fmt.Println(cfg.UseTLS())
-	case "address":
-		fmt.Println(cfg.Address())
-	case "user":
-		if cfg.User != "" {
-			fmt.Println(cfg.User)
-		}
-	case "password":
-		if cfg.User != "" {
-			fmt.Println(cfg.Password)
-		}
-	case "apikey":
-		if cfg.APIKey != "" {
-			fmt.Println(cfg.APIKey)
-		}
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
-	}
-}
-
-func guiSet(c *cli.Context) {
-	cfg := getConfig(c)
-	arg := c.Args()[0]
-	val := c.Args()[1]
-	switch strings.ToLower(arg) {
-	case "enabled":
-		cfg.GUI.Enabled = parseBool(val)
-	case "tls":
-		cfg.GUI.RawUseTLS = parseBool(val)
-	case "address":
-		validAddress(val)
-		cfg.GUI.RawAddress = val
-	case "user":
-		cfg.GUI.User = val
-	case "password":
-		cfg.GUI.Password = val
-	case "apikey":
-		cfg.GUI.APIKey = val
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
-	}
-	setConfig(c, cfg)
-}
-
-func guiUnset(c *cli.Context) {
-	cfg := getConfig(c)
-	arg := c.Args()[0]
-	switch strings.ToLower(arg) {
-	case "user":
-		cfg.GUI.User = ""
-	case "password":
-		cfg.GUI.Password = ""
-	case "apikey":
-		cfg.GUI.APIKey = ""
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: user, password, apikey")
-	}
-	setConfig(c, cfg)
-}

cmd/stcli/cmd_options.go

@@ -1,173 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "options",
-		HideHelp: true,
-		Usage:    "Options command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "dump",
-				Usage:    "Show all Syncthing option settings",
-				Requires: &cli.Requires{},
-				Action:   optionsDump,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a Syncthing option setting",
-				Requires: &cli.Requires{"setting"},
-				Action:   optionsGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a Syncthing option setting",
-				Requires: &cli.Requires{"setting", "value..."},
-				Action:   optionsSet,
-			},
-		},
-	})
-}
-
-func optionsDump(c *cli.Context) {
-	cfg := getConfig(c).Options
-	writer := newTableWriter()
-
-	fmt.Fprintln(writer, "Sync protocol listen addresses:\t", strings.Join(cfg.ListenAddresses, " "), "\t(addresses)")
-	fmt.Fprintln(writer, "Global discovery enabled:\t", cfg.GlobalAnnEnabled, "\t(globalannenabled)")
-	fmt.Fprintln(writer, "Global discovery servers:\t", strings.Join(cfg.GlobalAnnServers, " "), "\t(globalannserver)")
-
-	fmt.Fprintln(writer, "Local discovery enabled:\t", cfg.LocalAnnEnabled, "\t(localannenabled)")
-	fmt.Fprintln(writer, "Local discovery port:\t", cfg.LocalAnnPort, "\t(localannport)")
-
-	fmt.Fprintln(writer, "Outgoing rate limit in KiB/s:\t", cfg.MaxSendKbps, "\t(maxsend)")
-	fmt.Fprintln(writer, "Incoming rate limit in KiB/s:\t", cfg.MaxRecvKbps, "\t(maxrecv)")
-	fmt.Fprintln(writer, "Reconnect interval in seconds:\t", cfg.ReconnectIntervalS, "\t(reconnect)")
-	fmt.Fprintln(writer, "Start browser:\t", cfg.StartBrowser, "\t(browser)")
-	fmt.Fprintln(writer, "Enable UPnP:\t", cfg.NATEnabled, "\t(nat)")
-	fmt.Fprintln(writer, "UPnP Lease in minutes:\t", cfg.NATLeaseM, "\t(natlease)")
-	fmt.Fprintln(writer, "UPnP Renewal period in minutes:\t", cfg.NATRenewalM, "\t(natrenew)")
-	fmt.Fprintln(writer, "Restart on Wake Up:\t", cfg.RestartOnWakeup, "\t(wake)")
-
-	reporting := "unrecognized value"
-	switch cfg.URAccepted {
-	case -1:
-		reporting = "false"
-	case 0:
-		reporting = "undecided/false"
-	case 1:
-		reporting = "true"
-	}
-	fmt.Fprintln(writer, "Anonymous usage reporting:\t", reporting, "\t(reporting)")
-
-	writer.Flush()
-}
-
-func optionsGet(c *cli.Context) {
-	cfg := getConfig(c).Options
-	arg := c.Args()[0]
-	switch strings.ToLower(arg) {
-	case "address":
-		fmt.Println(strings.Join(cfg.ListenAddresses, "\n"))
-	case "globalannenabled":
-		fmt.Println(cfg.GlobalAnnEnabled)
-	case "globalannservers":
-		fmt.Println(strings.Join(cfg.GlobalAnnServers, "\n"))
-	case "localannenabled":
-		fmt.Println(cfg.LocalAnnEnabled)
-	case "localannport":
-		fmt.Println(cfg.LocalAnnPort)
-	case "maxsend":
-		fmt.Println(cfg.MaxSendKbps)
-	case "maxrecv":
-		fmt.Println(cfg.MaxRecvKbps)
-	case "reconnect":
-		fmt.Println(cfg.ReconnectIntervalS)
-	case "browser":
-		fmt.Println(cfg.StartBrowser)
-	case "nat":
-		fmt.Println(cfg.NATEnabled)
-	case "natlease":
-		fmt.Println(cfg.NATLeaseM)
-	case "natrenew":
-		fmt.Println(cfg.NATRenewalM)
-	case "reporting":
-		switch cfg.URAccepted {
-		case -1:
-			fmt.Println("false")
-		case 0:
-			fmt.Println("undecided/false")
-		case 1:
-			fmt.Println("true")
-		default:
-			fmt.Println("unknown")
-		}
-	case "wake":
-		fmt.Println(cfg.RestartOnWakeup)
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
-	}
-}
-
-func optionsSet(c *cli.Context) {
-	config := getConfig(c)
-	arg := c.Args()[0]
-	val := c.Args()[1]
-	switch strings.ToLower(arg) {
-	case "address":
-		for _, item := range c.Args().Tail() {
-			validAddress(item)
-		}
-		config.Options.ListenAddresses = c.Args().Tail()
-	case "globalannenabled":
-		config.Options.GlobalAnnEnabled = parseBool(val)
-	case "globalannserver":
-		for _, item := range c.Args().Tail() {
-			validAddress(item)
-		}
-		config.Options.GlobalAnnServers = c.Args().Tail()
-	case "localannenabled":
-		config.Options.LocalAnnEnabled = parseBool(val)
-	case "localannport":
-		config.Options.LocalAnnPort = parsePort(val)
-	case "maxsend":
-		config.Options.MaxSendKbps = parseUint(val)
-	case "maxrecv":
-		config.Options.MaxRecvKbps = parseUint(val)
-	case "reconnect":
-		config.Options.ReconnectIntervalS = parseUint(val)
-	case "browser":
-		config.Options.StartBrowser = parseBool(val)
-	case "nat":
-		config.Options.NATEnabled = parseBool(val)
-	case "natlease":
-		config.Options.NATLeaseM = parseUint(val)
-	case "natrenew":
-		config.Options.NATRenewalM = parseUint(val)
-	case "reporting":
-		switch strings.ToLower(val) {
-		case "u", "undecided", "unset":
-			config.Options.URAccepted = 0
-		default:
-			boolvalue := parseBool(val)
-			if boolvalue {
-				config.Options.URAccepted = 1
-			} else {
-				config.Options.URAccepted = -1
-			}
-		}
-	case "wake":
-		config.Options.RestartOnWakeup = parseBool(val)
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
-	}
-	setConfig(c, config)
-}

cmd/stcli/cmd_report.go

@@ -1,72 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "report",
-		HideHelp: true,
-		Usage:    "Reporting command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "system",
-				Usage:    "Report system state",
-				Requires: &cli.Requires{},
-				Action:   reportSystem,
-			},
-			{
-				Name:     "connections",
-				Usage:    "Report about connections to other devices",
-				Requires: &cli.Requires{},
-				Action:   reportConnections,
-			},
-			{
-				Name:     "usage",
-				Usage:    "Usage report",
-				Requires: &cli.Requires{},
-				Action:   reportUsage,
-			},
-		},
-	})
-}
-
-func reportSystem(c *cli.Context) {
-	response := httpGet(c, "system/status")
-	data := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &data)
-	prettyPrintJSON(data)
-}
-
-func reportConnections(c *cli.Context) {
-	response := httpGet(c, "system/connections")
-	data := make(map[string]map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &data)
-	var overall map[string]interface{}
-	for key, value := range data {
-		if key == "total" {
-			overall = value
-			continue
-		}
-		value["Device ID"] = key
-		prettyPrintJSON(value)
-		fmt.Println()
-	}
-	if overall != nil {
-		fmt.Println("=== Overall statistics ===")
-		prettyPrintJSON(overall)
-	}
-}
-
-func reportUsage(c *cli.Context) {
-	response := httpGet(c, "svc/report")
-	report := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &report)
-	prettyPrintJSON(report)
-}

cmd/stcli/errors.go

@@ -0,0 +1,61 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/urfave/cli"
+)
+
+var errorsCommand = cli.Command{
+	Name:     "errors",
+	HideHelp: true,
+	Usage:    "Error command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "show",
+			Usage:  "Show pending errors",
+			Action: expects(0, dumpOutput("system/error")),
+		},
+		{
+			Name:      "push",
+			Usage:     "Push an error to active clients",
+			ArgsUsage: "[error message]",
+			Action:    expects(1, errorsPush),
+		},
+		{
+			Name:   "clear",
+			Usage:  "Clear pending errors",
+			Action: expects(0, emptyPost("system/error/clear")),
+		},
+	},
+}
+
+func errorsPush(c *cli.Context) error {
+	client := c.App.Metadata["client"].(*APIClient)
+	errStr := strings.Join(c.Args(), " ")
+	response, err := client.Post("system/error", strings.TrimSpace(errStr))
+	if err != nil {
+		return err
+	}
+	if response.StatusCode != 200 {
+		errStr = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
+		bytes, err := responseToBArray(response)
+		if err != nil {
+			return err
+		}
+		body := string(bytes)
+		if body != "" {
+			errStr += "\nBody: " + body
+		}
+		return errors.New(errStr)
+	}
+	return nil
+}

cmd/stcli/labels.go

@@ -1,31 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-var jsonAttributeLabels = map[string]string{
-	"folderMaxMiB":   "Largest folder size in MiB",
-	"folderMaxFiles": "Largest folder file count",
-	"longVersion":    "Long version",
-	"totMiB":         "Total size in MiB",
-	"totFiles":       "Total files",
-	"uniqueID":       "Unique ID",
-	"numFolders":     "Folder count",
-	"numDevices":     "Device count",
-	"memoryUsageMiB": "Memory usage in MiB",
-	"memorySize":     "Total memory in MiB",
-	"sha256Perf":     "SHA256 Benchmark",
-	"At":             "Last contacted",
-	"Completion":     "Percent complete",
-	"InBytesTotal":   "Total bytes received",
-	"OutBytesTotal":  "Total bytes sent",
-	"ClientVersion":  "Client version",
-	"alloc":          "Memory allocated in bytes",
-	"sys":            "Memory using in bytes",
-	"cpuPercent":     "CPU load in percent",
-	"extAnnounceOK":  "External announcments working",
-	"goroutines":     "Number of Go routines",
-	"myID":           "Client ID",
-	"tilde":          "Tilde expands to",
-	"arch":           "Architecture",
-	"os":             "OS",
-}

cmd/stcli/main.go

@@ -1,63 +1,192 @@
-// Copyright (C) 2014 Audrius Butkevičius
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
-	"sort"
+	"bufio"
+	"crypto/tls"
+	"encoding/json"
+	"flag"
+	"log"
+	"os"
+	"reflect"
 
-	"github.com/AudriusButkevicius/cli"
+	"github.com/AudriusButkevicius/recli"
+	"github.com/flynn-archive/go-shlex"
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/locations"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/urfave/cli"
 )
 
-type ByAlphabet []cli.Command
-
-func (a ByAlphabet) Len() int           { return len(a) }
-func (a ByAlphabet) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
-func (a ByAlphabet) Less(i, j int) bool { return a[i].Name < a[j].Name }
-
-var cliCommands []cli.Command
-
 func main() {
-	app := cli.NewApp()
-	app.Name = "syncthing-cli"
-	app.Author = "Audrius Butkevičius"
-	app.Email = "audrius.butkevicius@gmail.com"
-	app.Usage = "Syncthing command line interface"
-	app.Version = "0.1"
-	app.HideHelp = true
+	// This is somewhat a hack around a chicken and egg problem.
+	// We need to set the home directory and potentially other flags to know where the syncthing instance is running
+	// in order to get it's config ... which we then use to construct the actual CLI ... at which point it's too late
+	// to add flags there...
+	homeBaseDir := locations.GetBaseDir(locations.ConfigBaseDir)
+	guiCfg := config.GUIConfiguration{}
 
-	app.Flags = []cli.Flag{
+	flags := flag.NewFlagSet("", flag.ContinueOnError)
+	flags.StringVar(&guiCfg.RawAddress, "gui-address", guiCfg.RawAddress, "Override GUI address (e.g. \"http://192.0.2.42:8443\")")
+	flags.StringVar(&guiCfg.APIKey, "gui-apikey", guiCfg.APIKey, "Override GUI API key")
+	flags.StringVar(&homeBaseDir, "home", homeBaseDir, "Set configuration directory")
+
+	// Implement the same flags at the lower CLI, with the same default values (pre-parse), but do nothing with them.
+	// This is so that we could reuse os.Args
+	fakeFlags := []cli.Flag{
 		cli.StringFlag{
-			Name:   "endpoint, e",
-			Value:  "http://127.0.0.1:8384",
-			Usage:  "End point to connect to",
-			EnvVar: "STENDPOINT",
+			Name:  "gui-address",
+			Value: guiCfg.RawAddress,
+			Usage: "Override GUI address (e.g. \"http://192.0.2.42:8443\")",
 		},
 		cli.StringFlag{
-			Name:   "apikey, k",
-			Value:  "",
-			Usage:  "API Key",
-			EnvVar: "STAPIKEY",
+			Name:  "gui-apikey",
+			Value: guiCfg.APIKey,
+			Usage: "Override GUI API key",
 		},
 		cli.StringFlag{
-			Name:   "username, u",
-			Value:  "",
-			Usage:  "Username",
-			EnvVar: "STUSERNAME",
-		},
-		cli.StringFlag{
-			Name:   "password, p",
-			Value:  "",
-			Usage:  "Password",
-			EnvVar: "STPASSWORD",
-		},
-		cli.BoolFlag{
-			Name:   "insecure, i",
-			Usage:  "Do not verify SSL certificate",
-			EnvVar: "STINSECURE",
+			Name:  "home",
+			Value: homeBaseDir,
+			Usage: "Set configuration directory",
 		},
 	}
 
-	sort.Sort(ByAlphabet(cliCommands))
-	app.Commands = cliCommands
-	app.RunAndExitOnError()
+	// Do not print usage of these flags, and ignore errors as this can't understand plenty of things
+	flags.Usage = func() {}
+	_ = flags.Parse(os.Args[1:])
+
+	// Now if the API key and address is not provided (we are not connecting to a remote instance),
+	// try to rip it out of the config.
+	if guiCfg.RawAddress == "" && guiCfg.APIKey == "" {
+		// Update the base directory
+		err := locations.SetBaseDir(locations.ConfigBaseDir, homeBaseDir)
+		if err != nil {
+			log.Fatal(errors.Wrap(err, "setting home"))
+		}
+
+		// Load the certs and get the ID
+		cert, err := tls.LoadX509KeyPair(
+			locations.Get(locations.CertFile),
+			locations.Get(locations.KeyFile),
+		)
+		if err != nil {
+			log.Fatal(errors.Wrap(err, "reading device ID"))
+		}
+
+		myID := protocol.NewDeviceID(cert.Certificate[0])
+
+		// Load the config
+		cfg, _, err := config.Load(locations.Get(locations.ConfigFile), myID, events.NoopLogger)
+		if err != nil {
+			log.Fatalln(errors.Wrap(err, "loading config"))
+		}
+
+		guiCfg = cfg.GUI()
+	} else if guiCfg.Address() == "" || guiCfg.APIKey == "" {
+		log.Fatalln("Both -gui-address and -gui-apikey should be specified")
+	}
+
+	if guiCfg.Address() == "" {
+		log.Fatalln("Could not find GUI Address")
+	}
+
+	if guiCfg.APIKey == "" {
+		log.Fatalln("Could not find GUI API key")
+	}
+
+	client := getClient(guiCfg)
+
+	cfg, err := getConfig(client)
+	original := cfg.Copy()
+	if err != nil {
+		log.Fatalln(errors.Wrap(err, "getting config"))
+	}
+
+	// Copy the config and set the default flags
+	recliCfg := recli.DefaultConfig
+	recliCfg.IDTag.Name = "xml"
+	recliCfg.SkipTag.Name = "json"
+
+	commands, err := recli.New(recliCfg).Construct(&cfg)
+	if err != nil {
+		log.Fatalln(errors.Wrap(err, "config reflect"))
+	}
+
+	// Construct the actual CLI
+	app := cli.NewApp()
+	app.Name = "stcli"
+	app.HelpName = app.Name
+	app.Author = "The Syncthing Authors"
+	app.Usage = "Syncthing command line interface"
+	app.Version = build.Version
+	app.Flags = fakeFlags
+	app.Metadata = map[string]interface{}{
+		"client": client,
+	}
+	app.Commands = []cli.Command{
+		{
+			Name:        "config",
+			HideHelp:    true,
+			Usage:       "Configuration modification command group",
+			Subcommands: commands,
+		},
+		showCommand,
+		operationCommand,
+		errorsCommand,
+	}
+
+	tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
+	if !tty {
+		// Not a TTY, consume from stdin
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			input, err := shlex.Split(scanner.Text())
+			if err != nil {
+				log.Fatalln(errors.Wrap(err, "parsing input"))
+			}
+			if len(input) == 0 {
+				continue
+			}
+			err = app.Run(append(os.Args, input...))
+			if err != nil {
+				log.Fatalln(err)
+			}
+		}
+		err = scanner.Err()
+		if err != nil {
+			log.Fatalln(err)
+		}
+	} else {
+		err = app.Run(os.Args)
+		if err != nil {
+			log.Fatalln(err)
+		}
+	}
+
+	if !reflect.DeepEqual(cfg, original) {
+		body, err := json.MarshalIndent(cfg, "", "  ")
+		if err != nil {
+			log.Fatalln(err)
+		}
+		resp, err := client.Post("system/config", string(body))
+		if err != nil {
+			log.Fatalln(err)
+		}
+		if resp.StatusCode != 200 {
+			body, err := responseToBArray(resp)
+			if err != nil {
+				log.Fatalln(err)
+			}
+			log.Fatalln(string(body))
+		}
+	}
 }

cmd/stcli/operations.go

@@ -0,0 +1,78 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli"
+)
+
+var operationCommand = cli.Command{
+	Name:     "operations",
+	HideHelp: true,
+	Usage:    "Operation command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "restart",
+			Usage:  "Restart syncthing",
+			Action: expects(0, emptyPost("system/restart")),
+		},
+		{
+			Name:   "shutdown",
+			Usage:  "Shutdown syncthing",
+			Action: expects(0, emptyPost("system/shutdown")),
+		},
+		{
+			Name:   "reset",
+			Usage:  "Reset syncthing deleting all folders and devices",
+			Action: expects(0, emptyPost("system/reset")),
+		},
+		{
+			Name:   "upgrade",
+			Usage:  "Upgrade syncthing (if a newer version is available)",
+			Action: expects(0, emptyPost("system/upgrade")),
+		},
+		{
+			Name:      "folder-override",
+			Usage:     "Override changes on folder (remote for sendonly, local for receiveonly)",
+			ArgsUsage: "[folder id]",
+			Action:    expects(1, foldersOverride),
+		},
+	},
+}
+
+func foldersOverride(c *cli.Context) error {
+	client := c.App.Metadata["client"].(*APIClient)
+	cfg, err := getConfig(client)
+	if err != nil {
+		return err
+	}
+	rid := c.Args()[0]
+	for _, folder := range cfg.Folders {
+		if folder.ID == rid {
+			response, err := client.Post("db/override", "")
+			if err != nil {
+				return err
+			}
+			if response.StatusCode != 200 {
+				errStr := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
+				bytes, err := responseToBArray(response)
+				if err != nil {
+					return err
+				}
+				body := string(bytes)
+				if body != "" {
+					errStr += "\nBody: " + body
+				}
+				return fmt.Errorf(errStr)
+			}
+			return nil
+		}
+	}
+	return fmt.Errorf("Folder " + rid + " not found")
+}

cmd/stcli/show.go

@@ -0,0 +1,44 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"github.com/urfave/cli"
+)
+
+var showCommand = cli.Command{
+	Name:     "show",
+	HideHelp: true,
+	Usage:    "Show command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "version",
+			Usage:  "Show syncthing client version",
+			Action: expects(0, dumpOutput("system/version")),
+		},
+		{
+			Name:   "config-status",
+			Usage:  "Show configuration status, whether or not a restart is required for changes to take effect",
+			Action: expects(0, dumpOutput("system/config/insync")),
+		},
+		{
+			Name:   "system",
+			Usage:  "Show system status",
+			Action: expects(0, dumpOutput("system/status")),
+		},
+		{
+			Name:   "connections",
+			Usage:  "Report about connections to other devices",
+			Action: expects(0, dumpOutput("system/connections")),
+		},
+		{
+			Name:   "usage",
+			Usage:  "Show usage report",
+			Action: expects(0, dumpOutput("svc/report")),
+		},
+	},
+}

cmd/stcli/utils.go

@@ -1,4 +1,8 @@
-// Copyright (C) 2014 Audrius Butkevičius
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -8,158 +12,83 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"text/tabwriter"
-	"unicode"
 
-	"github.com/AudriusButkevicius/cli"
 	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/urfave/cli"
 )
 
-func responseToBArray(response *http.Response) []byte {
-	defer response.Body.Close()
+func responseToBArray(response *http.Response) ([]byte, error) {
 	bytes, err := ioutil.ReadAll(response.Body)
 	if err != nil {
-		die(err)
+		return nil, err
 	}
-	return bytes
+	return bytes, response.Body.Close()
 }
 
-func die(vals ...interface{}) {
-	if len(vals) > 1 || vals[0] != nil {
-		os.Stderr.WriteString(fmt.Sprintln(vals...))
-		os.Exit(1)
+func emptyPost(url string) cli.ActionFunc {
+	return func(c *cli.Context) error {
+		client := c.App.Metadata["client"].(*APIClient)
+		_, err := client.Post(url, "")
+		return err
 	}
 }
 
-func wrappedHTTPPost(url string) func(c *cli.Context) {
-	return func(c *cli.Context) {
-		httpPost(c, url, "")
-	}
-}
-
-func prettyPrintJSON(json map[string]interface{}) {
-	writer := newTableWriter()
-	remap := make(map[string]interface{})
-	for k, v := range json {
-		key, ok := jsonAttributeLabels[k]
-		if !ok {
-			key = firstUpper(k)
+func dumpOutput(url string) cli.ActionFunc {
+	return func(c *cli.Context) error {
+		client := c.App.Metadata["client"].(*APIClient)
+		response, err := client.Get(url)
+		if err != nil {
+			return err
 		}
-		remap[key] = v
+		return prettyPrintResponse(c, response)
 	}
+}
 
-	jsonKeys := make([]string, 0, len(remap))
-	for key := range remap {
-		jsonKeys = append(jsonKeys, key)
+func getConfig(c *APIClient) (config.Configuration, error) {
+	cfg := config.Configuration{}
+	response, err := c.Get("system/config")
+	if err != nil {
+		return cfg, err
 	}
-	sort.Strings(jsonKeys)
-	for _, k := range jsonKeys {
-		var value string
-		rvalue := remap[k]
-		switch rvalue.(type) {
-		case int, int16, int32, int64, uint, uint16, uint32, uint64, float32, float64:
-			value = fmt.Sprintf("%.0f", rvalue)
-		default:
-			value = fmt.Sprint(rvalue)
+	bytes, err := responseToBArray(response)
+	if err != nil {
+		return cfg, err
+	}
+	err = json.Unmarshal(bytes, &cfg)
+	if err == nil {
+		return cfg, err
+	}
+	return cfg, nil
+}
+
+func expects(n int, actionFunc cli.ActionFunc) cli.ActionFunc {
+	return func(ctx *cli.Context) error {
+		if ctx.NArg() != n {
+			plural := ""
+			if n != 1 {
+				plural = "s"
+			}
+			return fmt.Errorf("expected %d argument%s, got %d", n, plural, ctx.NArg())
 		}
-		if value == "" {
-			continue
-		}
-		fmt.Fprintln(writer, k+":\t"+value)
-	}
-	writer.Flush()
-}
-
-func firstUpper(str string) string {
-	for i, v := range str {
-		return string(unicode.ToUpper(v)) + str[i+1:]
-	}
-	return ""
-}
-
-func newTableWriter() *tabwriter.Writer {
-	writer := new(tabwriter.Writer)
-	writer.Init(os.Stdout, 0, 8, 0, '\t', 0)
-	return writer
-}
-
-func getMyID(c *cli.Context) string {
-	response := httpGet(c, "system/status")
-	data := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &data)
-	return data["myID"].(string)
-}
-
-func getConfig(c *cli.Context) config.Configuration {
-	response := httpGet(c, "system/config")
-	config := config.Configuration{}
-	json.Unmarshal(responseToBArray(response), &config)
-	return config
-}
-
-func setConfig(c *cli.Context, cfg config.Configuration) {
-	body, err := json.Marshal(cfg)
-	die(err)
-	response := httpPost(c, "system/config", string(body))
-	if response.StatusCode != 200 {
-		die("Unexpected status code", response.StatusCode)
+		return actionFunc(ctx)
 	}
 }
 
-func parseBool(input string) bool {
-	val, err := strconv.ParseBool(input)
+func prettyPrintJSON(data interface{}) error {
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	return enc.Encode(data)
+}
+
+func prettyPrintResponse(c *cli.Context, response *http.Response) error {
+	bytes, err := responseToBArray(response)
 	if err != nil {
-		die(input + " is not a valid value for a boolean")
+		return err
 	}
-	return val
-}
-
-func parseInt(input string) int {
-	val, err := strconv.ParseInt(input, 0, 64)
-	if err != nil {
-		die(input + " is not a valid value for an integer")
-	}
-	return int(val)
-}
-
-func parseUint(input string) int {
-	val, err := strconv.ParseUint(input, 0, 64)
-	if err != nil {
-		die(input + " is not a valid value for an unsigned integer")
-	}
-	return int(val)
-}
-
-func parsePort(input string) int {
-	port := parseUint(input)
-	if port < 1 || port > 65535 {
-		die(input + " is not a valid port\nExpected value between 1 and 65535")
-	}
-	return port
-}
-
-func validAddress(input string) {
-	tokens := strings.Split(input, ":")
-	if len(tokens) != 2 {
-		die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
-	}
-	matched, err := regexp.MatchString("^[a-zA-Z0-9]+([-a-zA-Z0-9.]+[-a-zA-Z0-9]+)?$", tokens[0])
-	die(err)
-	if !matched {
-		die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
-	}
-	parsePort(tokens[1])
-}
-
-func parseDeviceID(input string) protocol.DeviceID {
-	device, err := protocol.DeviceIDFromString(input)
-	if err != nil {
-		die(input + " is not a valid device id")
-	}
-	return device
+	var data interface{}
+	if err := json.Unmarshal(bytes, &data); err != nil {
+		return err
+	}
+	// TODO: Check flag for pretty print format
+	return prettyPrintJSON(data)
 }

cmd/stcompdirs/main.go

@@ -60,7 +60,7 @@ func compareDirectories(dirs ...string) error {
 				} else if res[i].name > res[0].name {
 					return fmt.Errorf("%s missing %v (present in %s)", dirs[i], res[0], dirs[0])
 				}
-				return fmt.Errorf("Mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
+				return fmt.Errorf("mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
 			}
 		}
 

cmd/stcrashreceiver/sentry.go

@@ -0,0 +1,219 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"regexp"
+	"strings"
+	"sync"
+
+	raven "github.com/getsentry/raven-go"
+	"github.com/maruel/panicparse/stack"
+)
+
+const reportServer = "https://crash.syncthing.net/report/"
+
+var loader = newGithubSourceCodeLoader()
+
+func init() {
+	raven.SetSourceCodeLoader(loader)
+}
+
+var (
+	clients    = make(map[string]*raven.Client)
+	clientsMut sync.Mutex
+)
+
+func sendReport(dsn, path string, report []byte, userID string) error {
+	pkt, err := parseReport(path, report)
+	if err != nil {
+		return err
+	}
+
+	pkt.Interfaces = append(pkt.Interfaces, &raven.User{ID: userID})
+
+	clientsMut.Lock()
+	defer clientsMut.Unlock()
+
+	cli, ok := clients[dsn]
+	if !ok {
+		cli, err = raven.New(dsn)
+		if err != nil {
+			return err
+		}
+		clients[dsn] = cli
+	}
+
+	// The client sets release and such on the packet before sending, in the
+	// misguided idea that it knows this better than than the packet we give
+	// it. So we copy the values from the packet to the client first...
+	cli.SetRelease(pkt.Release)
+	cli.SetEnvironment(pkt.Environment)
+
+	defer cli.Wait()
+	_, errC := cli.Capture(pkt, nil)
+	return <-errC
+}
+
+func parseReport(path string, report []byte) (*raven.Packet, error) {
+	parts := bytes.SplitN(report, []byte("\n"), 2)
+	if len(parts) != 2 {
+		return nil, errors.New("no first line")
+	}
+
+	version, err := parseVersion(string(parts[0]))
+	if err != nil {
+		return nil, err
+	}
+	report = parts[1]
+
+	foundPanic := false
+	var subjectLine []byte
+	for {
+		parts = bytes.SplitN(report, []byte("\n"), 2)
+		if len(parts) != 2 {
+			return nil, errors.New("no panic line found")
+		}
+
+		line := parts[0]
+		report = parts[1]
+
+		if foundPanic {
+			// The previous line was our "Panic at ..." header. We are now
+			// at the beginning of the real panic trace and this is our
+			// subject line.
+			subjectLine = line
+			break
+		} else if bytes.HasPrefix(line, []byte("Panic at")) {
+			foundPanic = true
+		}
+	}
+
+	r := bytes.NewReader(report)
+	ctx, err := stack.ParseDump(r, ioutil.Discard, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// Lock the source code loader to the version we are processing here.
+	if version.commit != "" {
+		// We have a commit hash, so we know exactly which source to use
+		loader.LockWithVersion(version.commit)
+	} else if strings.HasPrefix(version.tag, "v") {
+		// Lets hope the tag is close enough
+		loader.LockWithVersion(version.tag)
+	} else {
+		// Last resort
+		loader.LockWithVersion("main")
+	}
+	defer loader.Unlock()
+
+	var trace raven.Stacktrace
+	for _, gr := range ctx.Goroutines {
+		if gr.First {
+			trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
+			for i, sc := range gr.Stack.Calls {
+				trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name(), sc.SrcPath, sc.Line, 3, nil)
+			}
+			break
+		}
+	}
+
+	pkt := &raven.Packet{
+		Message:     string(subjectLine),
+		Platform:    "go",
+		Release:     version.tag,
+		Environment: version.environment(),
+		Tags: raven.Tags{
+			raven.Tag{Key: "version", Value: version.version},
+			raven.Tag{Key: "tag", Value: version.tag},
+			raven.Tag{Key: "codename", Value: version.codename},
+			raven.Tag{Key: "runtime", Value: version.runtime},
+			raven.Tag{Key: "goos", Value: version.goos},
+			raven.Tag{Key: "goarch", Value: version.goarch},
+			raven.Tag{Key: "builder", Value: version.builder},
+		},
+		Extra: raven.Extra{
+			"url": reportServer + path,
+		},
+		Interfaces: []raven.Interface{&trace},
+	}
+	if version.commit != "" {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
+	}
+	for _, tag := range version.extra {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: tag, Value: "1"})
+	}
+
+	return pkt, nil
+}
+
+// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]
+var longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
+
+type version struct {
+	version  string   // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
+	tag      string   // "v1.1.4-rc.1"
+	commit   string   // "6aaae618", blank when absent
+	codename string   // "Erbium Earthworm"
+	runtime  string   // "go1.12.5"
+	goos     string   // "darwin"
+	goarch   string   // "amd64"
+	builder  string   // "jb@kvin.kastelo.net"
+	extra    []string // "foo", "bar"
+}
+
+func (v version) environment() string {
+	if v.commit != "" {
+		return "Development"
+	}
+	if strings.Contains(v.tag, "-rc.") {
+		return "Candidate"
+	}
+	if strings.Contains(v.tag, "-") {
+		return "Beta"
+	}
+	return "Stable"
+}
+
+func parseVersion(line string) (version, error) {
+	m := longVersionRE.FindStringSubmatch(line)
+	if len(m) == 0 {
+		return version{}, errors.New("unintelligeble version string")
+	}
+
+	v := version{
+		version:  m[1],
+		codename: m[2],
+		runtime:  m[3],
+		goos:     m[4],
+		goarch:   m[5],
+		builder:  m[6],
+	}
+
+	parts := strings.Split(v.version, "+")
+	v.tag = parts[0]
+	if len(parts) > 1 {
+		fields := strings.Split(parts[1], "-")
+		if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
+			v.commit = fields[1][1:]
+		}
+	}
+
+	if len(m) >= 8 && m[7] != "" {
+		tags := strings.Split(m[7], ",")
+		for i := range tags {
+			tags[i] = strings.TrimSpace(tags[i])
+		}
+		v.extra = tags
+	}
+
+	return v, nil
+}

cmd/stcrashreceiver/sentry_test.go

@@ -0,0 +1,78 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+)
+
+func TestParseVersion(t *testing.T) {
+	cases := []struct {
+		longVersion string
+		parsed      version
+	}{
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+			},
+		},
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+				extra:    []string{"foo", "bar"},
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		v, err := parseVersion(tc.longVersion)
+		if err != nil {
+			t.Errorf("%s\nerror: %v\n", tc.longVersion, err)
+			continue
+		}
+		if fmt.Sprint(v) != fmt.Sprint(tc.parsed) {
+			t.Errorf("%s\nA: %v\nE: %v\n", tc.longVersion, v, tc.parsed)
+		}
+	}
+}
+
+func TestParseReport(t *testing.T) {
+	bs, err := ioutil.ReadFile("_testdata/panic.log")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pkt, err := parseReport("1/2/345", bs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bs, err = pkt.JSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Printf("%s\n", bs)
+}

cmd/stcrashreceiver/sourcecodeloader.go

@@ -0,0 +1,118 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	urlPrefix   = "https://raw.githubusercontent.com/syncthing/syncthing/"
+	httpTimeout = 10 * time.Second
+)
+
+type githubSourceCodeLoader struct {
+	mut     sync.Mutex
+	version string
+	cache   map[string]map[string][][]byte // version -> file -> lines
+	client  *http.Client
+}
+
+func newGithubSourceCodeLoader() *githubSourceCodeLoader {
+	return &githubSourceCodeLoader{
+		cache:  make(map[string]map[string][][]byte),
+		client: &http.Client{Timeout: httpTimeout},
+	}
+}
+
+func (l *githubSourceCodeLoader) LockWithVersion(version string) {
+	l.mut.Lock()
+	l.version = version
+	if _, ok := l.cache[version]; !ok {
+		l.cache[version] = make(map[string][][]byte)
+	}
+}
+
+func (l *githubSourceCodeLoader) Unlock() {
+	l.mut.Unlock()
+}
+
+func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]byte, int) {
+	filename = filepath.ToSlash(filename)
+	lines, ok := l.cache[l.version][filename]
+	if !ok {
+		// Cache whatever we managed to find (or nil if nothing, so we don't try again)
+		defer func() {
+			l.cache[l.version][filename] = lines
+		}()
+
+		knownPrefixes := []string{"/lib/", "/cmd/"}
+		var idx int
+		for _, pref := range knownPrefixes {
+			idx = strings.Index(filename, pref)
+			if idx >= 0 {
+				break
+			}
+		}
+		if idx == -1 {
+			return nil, 0
+		}
+
+		url := urlPrefix + l.version + filename[idx:]
+		resp, err := l.client.Get(url)
+
+		if err != nil {
+			fmt.Println("Loading source:", err)
+			return nil, 0
+		}
+		if resp.StatusCode != http.StatusOK {
+			fmt.Println("Loading source:", resp.Status)
+			return nil, 0
+		}
+		data, err := ioutil.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		if err != nil {
+			fmt.Println("Loading source:", err.Error())
+			return nil, 0
+		}
+		lines = bytes.Split(data, []byte{'\n'})
+	}
+
+	return getLineFromLines(lines, line, context)
+}
+
+func getLineFromLines(lines [][]byte, line, context int) ([][]byte, int) {
+	if lines == nil {
+		// cached error from ReadFile: return no lines
+		return nil, 0
+	}
+
+	line-- // stack trace lines are 1-indexed
+	start := line - context
+	var idx int
+	if start < 0 {
+		start = 0
+		idx = line
+	} else {
+		idx = context
+	}
+	end := line + context + 1
+	if line >= len(lines) {
+		return nil, 0
+	}
+	if end > len(lines) {
+		end = len(lines)
+	}
+	return lines[start:end], idx
+}

cmd/stcrashreceiver/stcrashreceiver.go

@@ -0,0 +1,185 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Command stcrashreceiver is a trivial HTTP server that allows two things:
+//
+// - uploading files (crash reports) named like a SHA256 hash using a PUT request
+// - checking whether such file exists using a HEAD request
+//
+// Typically this should be deployed behind something that manages HTTPS.
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/sha256"
+)
+
+const maxRequestSize = 1 << 20 // 1 MiB
+
+func main() {
+	dir := flag.String("dir", ".", "Directory to store reports in")
+	dsn := flag.String("dsn", "", "Sentry DSN")
+	listen := flag.String("listen", ":22039", "HTTP listen address")
+	flag.Parse()
+
+	cr := &crashReceiver{
+		dir: *dir,
+		dsn: *dsn,
+	}
+
+	log.SetOutput(os.Stdout)
+	if err := http.ListenAndServe(*listen, cr); err != nil {
+		log.Fatalln("HTTP serve:", err)
+	}
+}
+
+type crashReceiver struct {
+	dir string
+	dsn string
+}
+
+func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	// The final path component should be a SHA256 hash in hex, so 64 hex
+	// characters. We don't care about case on the request but use lower
+	// case internally.
+	reportID := strings.ToLower(path.Base(req.URL.Path))
+	if len(reportID) != 64 {
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+	for _, c := range reportID {
+		if c >= 'a' && c <= 'f' {
+			continue
+		}
+		if c >= '0' && c <= '9' {
+			continue
+		}
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+
+	// The location of the report on disk, compressed
+	fullPath := filepath.Join(r.dir, r.dirFor(reportID), reportID) + ".gz"
+
+	switch req.Method {
+	case http.MethodGet:
+		r.serveGet(fullPath, w, req)
+	case http.MethodHead:
+		r.serveHead(fullPath, w, req)
+	case http.MethodPut:
+		r.servePut(reportID, fullPath, w, req)
+	default:
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+// serveGet responds to GET requests by serving the uncompressed report.
+func (r *crashReceiver) serveGet(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	fd, err := os.Open(fullPath)
+	if err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+		return
+	}
+
+	defer fd.Close()
+	gr, err := gzip.NewReader(fd)
+	if err != nil {
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+	_, _ = io.Copy(w, gr) // best effort
+}
+
+// serveHead responds to HEAD requests by checking if the named report
+// already exists in the system.
+func (r *crashReceiver) serveHead(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	if _, err := os.Lstat(fullPath); err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+	}
+}
+
+// servePut accepts and stores the given report.
+func (r *crashReceiver) servePut(reportID, fullPath string, w http.ResponseWriter, req *http.Request) {
+	// Ensure the destination directory exists
+	if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
+		log.Println("Creating directory:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Read at most maxRequestSize of report data.
+	log.Println("Receiving report", reportID)
+	lr := io.LimitReader(req.Body, maxRequestSize)
+	bs, err := ioutil.ReadAll(lr)
+	if err != nil {
+		log.Println("Reading report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Compress the report for storage
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	_, _ = gw.Write(bs) // can't fail
+	gw.Close()
+
+	// Create an output file with the compressed report
+	err = ioutil.WriteFile(fullPath, buf.Bytes(), 0644)
+	if err != nil {
+		log.Println("Saving report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Send the report to Sentry
+	if r.dsn != "" {
+		// Remote ID
+		user := userIDFor(req)
+
+		go func() {
+			// There's no need for the client to have to wait for this part.
+			if err := sendReport(r.dsn, reportID, bs, user); err != nil {
+				log.Println("Failed to send report:", err)
+			}
+		}()
+	}
+}
+
+// 01234567890abcdef... => 01/23
+func (r *crashReceiver) dirFor(base string) string {
+	return filepath.Join(base[0:2], base[2:4])
+}
+
+// userIDFor returns a string we can use as the user ID for the purpose of
+// counting affected users. It's the truncated hash of a salt, the user
+// remote IP, and the current month.
+func userIDFor(req *http.Request) string {
+	addr := req.RemoteAddr
+	if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
+		addr = fwd
+	}
+	if host, _, err := net.SplitHostPort(addr); err == nil {
+		addr = host
+	}
+	now := time.Now().Format("200601")
+	salt := "stcrashreporter"
+	hash := sha256.Sum256([]byte(salt + addr + now))
+	return fmt.Sprintf("%x", hash[:8])
+}

cmd/stdiscosrv/apisrv.go

@@ -8,6 +8,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
 	"encoding/json"
 	"encoding/pem"
@@ -17,13 +18,13 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/syncthing/syncthing/lib/protocol"
-	"golang.org/x/net/context"
 )
 
 // announcement is the format received from and sent to clients
@@ -131,11 +132,19 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 		log.Println(reqID, req.Method, req.URL)
 	}
 
-	var remoteIP net.IP
+	remoteAddr := &net.TCPAddr{
+		IP:   nil,
+		Port: -1,
+	}
+
 	if s.useHTTP {
-		remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
+		remoteAddr.IP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
+		if parsedPort, err := strconv.ParseInt(req.Header.Get("X-Client-Port"), 10, 0); err == nil {
+			remoteAddr.Port = int(parsedPort)
+		}
 	} else {
-		addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
+		var err error
+		remoteAddr, err = net.ResolveTCPAddr("tcp", req.RemoteAddr)
 		if err != nil {
 			log.Println("remoteAddr:", err)
 			lw.Header().Set("Retry-After", errorRetryAfterString())
@@ -143,14 +152,13 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 			apiRequestsTotal.WithLabelValues("no_remote_addr").Inc()
 			return
 		}
-		remoteIP = addr.IP
 	}
 
 	switch req.Method {
 	case "GET":
 		s.handleGET(ctx, lw, req)
 	case "POST":
-		s.handlePOST(ctx, remoteIP, lw, req)
+		s.handlePOST(ctx, remoteAddr, lw, req)
 	default:
 		http.Error(lw, "Method Not Allowed", http.StatusMethodNotAllowed)
 	}
@@ -195,6 +203,7 @@ func (s *apiSrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http
 
 		if misses%notFoundMissesWriteInterval == 0 {
 			rec.Misses = misses
+			rec.Missed = time.Now().UnixNano()
 			rec.Addresses = nil
 			// rec.Seen retained from get
 			s.db.put(key, rec)
@@ -215,7 +224,7 @@ func (s *apiSrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http
 	w.Write(bs)
 }
 
-func (s *apiSrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
+func (s *apiSrv) handlePOST(ctx context.Context, remoteAddr *net.TCPAddr, w http.ResponseWriter, req *http.Request) {
 	reqID := ctx.Value(idKey).(requestID)
 
 	rawCert := certificateBytes(req)
@@ -242,7 +251,7 @@ func (s *apiSrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.Respons
 
 	deviceID := protocol.NewDeviceID(rawCert)
 
-	addresses := fixupAddresses(remoteIP, ann.Addresses)
+	addresses := fixupAddresses(remoteAddr, ann.Addresses)
 	if len(addresses) == 0 {
 		announceRequestsTotal.WithLabelValues("bad_request").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
@@ -250,7 +259,7 @@ func (s *apiSrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.Respons
 		return
 	}
 
-	if err := s.handleAnnounce(remoteIP, deviceID, addresses); err != nil {
+	if err := s.handleAnnounce(deviceID, addresses); err != nil {
 		announceRequestsTotal.WithLabelValues("internal_error").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
@@ -267,7 +276,7 @@ func (s *apiSrv) Stop() {
 	s.listener.Close()
 }
 
-func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addresses []string) error {
+func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string) error {
 	key := deviceID.String()
 	now := time.Now()
 	expire := now.Add(addressExpiryTime).UnixNano()
@@ -278,6 +287,10 @@ func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addre
 		dbAddrs[i].Expires = expire
 	}
 
+	// The address slice must always be sorted for database merges to work
+	// properly.
+	sort.Sort(databaseAddressOrder(dbAddrs))
+
 	seen := now.UnixNano()
 	if s.repl != nil {
 		s.repl.send(key, dbAddrs, seen)
@@ -294,33 +307,71 @@ func certificateBytes(req *http.Request) []byte {
 		return req.TLS.PeerCertificates[0].Raw
 	}
 
+	var bs []byte
+
 	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
-		bs := []byte(hdr)
-		// The certificate is in PEM format but with spaces for newlines. We
-		// need to reinstate the newlines for the PEM decoder. But we need to
-		// leave the spaces in the BEGIN and END lines - the first and last
-		// space - alone.
-		firstSpace := bytes.Index(bs, []byte(" "))
-		lastSpace := bytes.LastIndex(bs, []byte(" "))
-		for i := firstSpace + 1; i < lastSpace; i++ {
-			if bs[i] == ' ' {
-				bs[i] = '\n'
+		if strings.Contains(hdr, "%") {
+			// Nginx using $ssl_client_escaped_cert
+			// The certificate is in PEM format with url encoding.
+			// We need to decode for the PEM decoder
+			hdr, err := url.QueryUnescape(hdr)
+			if err != nil {
+				// Decoding failed
+				return nil
+			}
+
+			bs = []byte(hdr)
+		} else {
+			// Nginx using $ssl_client_cert
+			// The certificate is in PEM format but with spaces for newlines. We
+			// need to reinstate the newlines for the PEM decoder. But we need to
+			// leave the spaces in the BEGIN and END lines - the first and last
+			// space - alone.
+			bs = []byte(hdr)
+			firstSpace := bytes.Index(bs, []byte(" "))
+			lastSpace := bytes.LastIndex(bs, []byte(" "))
+			for i := firstSpace + 1; i < lastSpace; i++ {
+				if bs[i] == ' ' {
+					bs[i] = '\n'
+				}
 			}
 		}
-		block, _ := pem.Decode(bs)
-		if block == nil {
+	} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
+		// Traefik 2 passtlsclientcert
+		// The certificate is in PEM format with url encoding but without newlines
+		// and start/end statements. We need to decode, reinstate the newlines every 64
+		// character and add statements for the PEM decoder
+		hdr, err := url.QueryUnescape(hdr)
+		if err != nil {
 			// Decoding failed
 			return nil
 		}
-		return block.Bytes
+
+		for i := 64; i < len(hdr); i += 65 {
+			hdr = hdr[:i] + "\n" + hdr[i:]
+		}
+
+		hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
+		hdr = hdr + "\n-----END CERTIFICATE-----\n"
+		bs = []byte(hdr)
 	}
 
-	return nil
+	if bs == nil {
+		return nil
+	}
+
+	block, _ := pem.Decode(bs)
+	if block == nil {
+		// Decoding failed
+		return nil
+	}
+
+	return block.Bytes
 }
 
 // fixupAddresses checks the list of addresses, removing invalid ones and
 // replacing unspecified IPs with the given remote IP.
-func fixupAddresses(remote net.IP, addresses []string) []string {
+func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
 	fixed := make([]string, 0, len(addresses))
 	for _, annAddr := range addresses {
 		uri, err := url.Parse(annAddr)
@@ -334,19 +385,40 @@ func fixupAddresses(remote net.IP, addresses []string) []string {
 		}
 
 		ip := net.ParseIP(host)
-		if host == "" || ip.IsUnspecified() {
-			// Do not use IPv6 remote address if requested scheme is ...4
-			// (i.e., tcp4, etc.)
-			if strings.HasSuffix(uri.Scheme, "4") && remote.To4() == nil {
-				continue
+
+		// Some classes of IP are no-go.
+		if ip.IsLoopback() || ip.IsMulticast() {
+			continue
+		}
+
+		if remote != nil {
+			if host == "" || ip.IsUnspecified() {
+				// Replace the unspecified IP with the request source.
+
+				// ... unless the request source is the loopback address or
+				// multicast/unspecified (can't happen, really).
+				if remote.IP.IsLoopback() || remote.IP.IsMulticast() || remote.IP.IsUnspecified() {
+					continue
+				}
+
+				// Do not use IPv6 remote address if requested scheme is ...4
+				// (i.e., tcp4, etc.)
+				if strings.HasSuffix(uri.Scheme, "4") && remote.IP.To4() == nil {
+					continue
+				}
+
+				// Do not use IPv4 remote address if requested scheme is ...6
+				if strings.HasSuffix(uri.Scheme, "6") && remote.IP.To4() != nil {
+					continue
+				}
+
+				host = remote.IP.String()
 			}
 
-			// Do not use IPv4 remote address if requested scheme is ...6
-			if strings.HasSuffix(uri.Scheme, "6") && remote.To4() != nil {
-				continue
+			// If zero port was specified, use remote port.
+			if port == "0" && remote.Port > 0 {
+				port = fmt.Sprintf("%d", remote.Port)
 			}
-
-			host = remote.String()
 		}
 
 		uri.Host = net.JoinHostPort(host, port)

cmd/stdiscosrv/apisrv_test.go

@@ -0,0 +1,88 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"net"
+	"testing"
+)
+
+func TestFixupAddresses(t *testing.T) {
+	cases := []struct {
+		remote *net.TCPAddr
+		in     []string
+		out    []string
+	}{
+		{ // verbatim passthrough
+			in:  []string{"tcp://1.2.3.4:22000"},
+			out: []string{"tcp://1.2.3.4:22000"},
+		}, { // unspecified replaced by remote
+			remote: addr("1.2.3.4", 22000),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://1.2.3.4:22000", "tcp://192.0.2.42:22000"},
+		}, { // unspecified not used as replacement
+			remote: addr("0.0.0.0", 22000),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // unspecified not used as replacement
+			remote: addr("::", 22000),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // localhost not used as replacement
+			remote: addr("127.0.0.1", 22000),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // localhost not used as replacement
+			remote: addr("::1", 22000),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // multicast not used as replacement
+			remote: addr("224.0.0.1", 22000),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // multicast not used as replacement
+			remote: addr("ff80::42", 22000),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // explicitly announced weirdness is also filtered
+			remote: addr("192.0.2.42", 22000),
+			in:     []string{"tcp://:22000", "tcp://127.1.2.3:22000", "tcp://[::1]:22000", "tcp://[ff80::42]:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // port remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://0.0.0.0:0"},
+			out:    []string{"tcp://123.123.123.123:9000"},
+		}, { // unspecified port remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://:0"},
+			out:    []string{"tcp://123.123.123.123:9000"},
+		}, { // empty remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://"},
+			out:    []string{},
+		}, { // port only remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://44.44.44.44:0"},
+			out:    []string{"tcp://44.44.44.44:9000"},
+		},
+	}
+
+	for _, tc := range cases {
+		out := fixupAddresses(tc.remote, tc.in)
+		if fmt.Sprint(out) != fmt.Sprint(tc.out) {
+			t.Errorf("fixupAddresses(%v, %v) => %v, expected %v", tc.remote, tc.in, out, tc.out)
+		}
+	}
+}
+
+func addr(host string, port int) *net.TCPAddr {
+	return &net.TCPAddr{
+		IP:   net.ParseIP(host),
+		Port: port,
+	}
+}

cmd/stdiscosrv/database.go

@@ -4,12 +4,13 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//go:generate go run ../../script/protofmt.go database.proto
-//go:generate protoc -I ../../../../../ -I ../../vendor/ -I ../../vendor/github.com/gogo/protobuf/protobuf -I . --gogofast_out=. database.proto
+//go:generate go run ../../proto/scripts/protofmt.go database.proto
+//go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
 
 package main
 
 import (
+	"log"
 	"sort"
 	"time"
 
@@ -162,10 +163,10 @@ func (s *levelDBStore) Serve() {
 	// Start the statistics serve routine. It will exit with us when
 	// statisticsTrigger is closed.
 	statisticsTrigger := make(chan struct{})
-	defer close(statisticsTrigger)
 	statisticsDone := make(chan struct{})
 	go s.statisticsServe(statisticsTrigger, statisticsDone)
 
+loop:
 	for {
 		select {
 		case fn := <-s.inbox:
@@ -184,17 +185,24 @@ func (s *levelDBStore) Serve() {
 
 		case <-s.stop:
 			// We're done.
-			return
+			close(statisticsTrigger)
+			break loop
 		}
 	}
+
+	// Also wait for statisticsServe to return
+	<-statisticsDone
 }
 
 func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
+	defer close(done)
+
 	for range trigger {
 		t0 := time.Now()
 		nowNanos := t0.UnixNano()
 		cutoff24h := t0.Add(-24 * time.Hour).UnixNano()
 		cutoff1w := t0.Add(-7 * 24 * time.Hour).UnixNano()
+		cutoff2Mon := t0.Add(-60 * 24 * time.Hour).UnixNano()
 		current, last24h, last1w, inactive, errors := 0, 0, 0, 0, 0
 
 		iter := s.db.NewIterator(&util.Range{}, nil)
@@ -217,6 +225,17 @@ func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- stru
 				last24h++
 			case rec.Seen > cutoff1w:
 				last1w++
+			case rec.Seen > cutoff2Mon:
+				inactive++
+			case rec.Missed < cutoff2Mon:
+				// It hasn't been seen lately and we haven't recorded
+				// someone asking for this device in a long time either;
+				// delete the record.
+				if err := s.db.Delete(iter.Key(), nil); err != nil {
+					databaseOperations.WithLabelValues(dbOpDelete, dbResError).Inc()
+				} else {
+					databaseOperations.WithLabelValues(dbOpDelete, dbResSuccess).Inc()
+				}
 			default:
 				inactive++
 			}
@@ -245,12 +264,15 @@ func (s *levelDBStore) Stop() {
 // chosen for any duplicates.
 func merge(a, b DatabaseRecord) DatabaseRecord {
 	// Both lists must be sorted for this to work.
-	sort.Slice(a.Addresses, func(i, j int) bool {
-		return a.Addresses[i].Address < a.Addresses[j].Address
-	})
-	sort.Slice(b.Addresses, func(i, j int) bool {
-		return b.Addresses[i].Address < b.Addresses[j].Address
-	})
+	if !sort.IsSorted(databaseAddressOrder(a.Addresses)) {
+		log.Println("Warning: bug: addresses not correctly sorted in merge")
+		a.Addresses = sortedAddressCopy(a.Addresses)
+	}
+	if !sort.IsSorted(databaseAddressOrder(b.Addresses)) {
+		// no warning because this is the side we read from disk and it may
+		// legitimately predate correct sorting.
+		b.Addresses = sortedAddressCopy(b.Addresses)
+	}
 
 	res := DatabaseRecord{
 		Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
@@ -334,3 +356,24 @@ func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
 	}
 	return addrs
 }
+
+func sortedAddressCopy(addrs []DatabaseAddress) []DatabaseAddress {
+	sorted := make([]DatabaseAddress, len(addrs))
+	copy(sorted, addrs)
+	sort.Sort(databaseAddressOrder(sorted))
+	return sorted
+}
+
+type databaseAddressOrder []DatabaseAddress
+
+func (s databaseAddressOrder) Less(a, b int) bool {
+	return s[a].Address < s[b].Address
+}
+
+func (s databaseAddressOrder) Swap(a, b int) {
+	s[a], s[b] = s[b], s[a]
+}
+
+func (s databaseAddressOrder) Len() int {
+	return len(s)
+}

cmd/stdiscosrv/database.pb.go

@@ -1,25 +1,16 @@
 // Code generated by protoc-gen-gogo. DO NOT EDIT.
 // source: database.proto
 
-/*
-	Package main is a generated protocol buffer package.
-
-	It is generated from these files:
-		database.proto
-
-	It has these top-level messages:
-		DatabaseRecord
-		ReplicationRecord
-		DatabaseAddress
-*/
 package main
 
-import proto "github.com/gogo/protobuf/proto"
-import fmt "fmt"
-import math "math"
-import _ "github.com/gogo/protobuf/gogoproto"
-
-import io "io"
+import (
+	fmt "fmt"
+	_ "github.com/gogo/protobuf/gogoproto"
+	proto "github.com/gogo/protobuf/proto"
+	io "io"
+	math "math"
+	math_bits "math/bits"
+)
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
@@ -30,49 +21,158 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
 
 type DatabaseRecord struct {
-	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses" json:"addresses"`
+	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
 	Misses    int32             `protobuf:"varint,2,opt,name=misses,proto3" json:"misses,omitempty"`
 	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
+	Missed    int64             `protobuf:"varint,4,opt,name=missed,proto3" json:"missed,omitempty"`
 }
 
-func (m *DatabaseRecord) Reset()                    { *m = DatabaseRecord{} }
-func (m *DatabaseRecord) String() string            { return proto.CompactTextString(m) }
-func (*DatabaseRecord) ProtoMessage()               {}
-func (*DatabaseRecord) Descriptor() ([]byte, []int) { return fileDescriptorDatabase, []int{0} }
+func (m *DatabaseRecord) Reset()         { *m = DatabaseRecord{} }
+func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
+func (*DatabaseRecord) ProtoMessage()    {}
+func (*DatabaseRecord) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{0}
+}
+func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseRecord.Merge(m, src)
+}
+func (m *DatabaseRecord) XXX_Size() int {
+	return m.Size()
+}
+func (m *DatabaseRecord) XXX_DiscardUnknown() {
+	xxx_messageInfo_DatabaseRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DatabaseRecord proto.InternalMessageInfo
 
 type ReplicationRecord struct {
 	Key       string            `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
-	Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses" json:"addresses"`
+	Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses"`
 	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
 }
 
-func (m *ReplicationRecord) Reset()                    { *m = ReplicationRecord{} }
-func (m *ReplicationRecord) String() string            { return proto.CompactTextString(m) }
-func (*ReplicationRecord) ProtoMessage()               {}
-func (*ReplicationRecord) Descriptor() ([]byte, []int) { return fileDescriptorDatabase, []int{1} }
+func (m *ReplicationRecord) Reset()         { *m = ReplicationRecord{} }
+func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
+func (*ReplicationRecord) ProtoMessage()    {}
+func (*ReplicationRecord) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{1}
+}
+func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ReplicationRecord.Merge(m, src)
+}
+func (m *ReplicationRecord) XXX_Size() int {
+	return m.Size()
+}
+func (m *ReplicationRecord) XXX_DiscardUnknown() {
+	xxx_messageInfo_ReplicationRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ReplicationRecord proto.InternalMessageInfo
 
 type DatabaseAddress struct {
 	Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
 	Expires int64  `protobuf:"varint,2,opt,name=expires,proto3" json:"expires,omitempty"`
 }
 
-func (m *DatabaseAddress) Reset()                    { *m = DatabaseAddress{} }
-func (m *DatabaseAddress) String() string            { return proto.CompactTextString(m) }
-func (*DatabaseAddress) ProtoMessage()               {}
-func (*DatabaseAddress) Descriptor() ([]byte, []int) { return fileDescriptorDatabase, []int{2} }
+func (m *DatabaseAddress) Reset()         { *m = DatabaseAddress{} }
+func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
+func (*DatabaseAddress) ProtoMessage()    {}
+func (*DatabaseAddress) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{2}
+}
+func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseAddress.Merge(m, src)
+}
+func (m *DatabaseAddress) XXX_Size() int {
+	return m.Size()
+}
+func (m *DatabaseAddress) XXX_DiscardUnknown() {
+	xxx_messageInfo_DatabaseAddress.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DatabaseAddress proto.InternalMessageInfo
 
 func init() {
 	proto.RegisterType((*DatabaseRecord)(nil), "main.DatabaseRecord")
 	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
 	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
 }
+
+func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
+
+var fileDescriptor_b90fe3356ea5df07 = []byte{
+	// 270 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
+	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
+	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
+	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
+	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
+	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
+	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
+	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
+	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
+	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
+	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
+	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
+	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
+	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
+	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
+	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
+	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
+}
+
 func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -80,39 +180,51 @@ func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Addresses) > 0 {
-		for _, msg := range m.Addresses {
-			dAtA[i] = 0xa
-			i++
-			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
-			n, err := msg.MarshalTo(dAtA[i:])
-			if err != nil {
-				return 0, err
-			}
-			i += n
-		}
-	}
-	if m.Misses != 0 {
-		dAtA[i] = 0x10
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+	if m.Missed != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+		i--
+		dAtA[i] = 0x20
 	}
 	if m.Seen != 0 {
-		dAtA[i] = 0x18
-		i++
 		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
 	}
-	return i, nil
+	if m.Misses != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+		i--
+		dAtA[i] = 0x10
+	}
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
 }
 
 func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -120,40 +232,48 @@ func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Key) > 0 {
-		dAtA[i] = 0xa
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
-		i += copy(dAtA[i:], m.Key)
+	if m.Seen != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
 	}
 	if len(m.Addresses) > 0 {
-		for _, msg := range m.Addresses {
-			dAtA[i] = 0x12
-			i++
-			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
-			n, err := msg.MarshalTo(dAtA[i:])
-			if err != nil {
-				return 0, err
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
 			}
-			i += n
+			i--
+			dAtA[i] = 0x12
 		}
 	}
-	if m.Seen != 0 {
-		dAtA[i] = 0x18
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+	if len(m.Key) > 0 {
+		i -= len(m.Key)
+		copy(dAtA[i:], m.Key)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		i--
+		dAtA[i] = 0xa
 	}
-	return i, nil
+	return len(dAtA) - i, nil
 }
 
 func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -161,52 +281,45 @@ func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Address) > 0 {
-		dAtA[i] = 0xa
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
-		i += copy(dAtA[i:], m.Address)
-	}
 	if m.Expires != 0 {
-		dAtA[i] = 0x10
-		i++
 		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
+		i--
+		dAtA[i] = 0x10
 	}
-	return i, nil
+	if len(m.Address) > 0 {
+		i -= len(m.Address)
+		copy(dAtA[i:], m.Address)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
 }
 
-func encodeFixed64Database(dAtA []byte, offset int, v uint64) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	dAtA[offset+4] = uint8(v >> 32)
-	dAtA[offset+5] = uint8(v >> 40)
-	dAtA[offset+6] = uint8(v >> 48)
-	dAtA[offset+7] = uint8(v >> 56)
-	return offset + 8
-}
-func encodeFixed32Database(dAtA []byte, offset int, v uint32) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	return offset + 4
-}
 func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
+	offset -= sovDatabase(v)
+	base := offset
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
 		v >>= 7
 		offset++
 	}
 	dAtA[offset] = uint8(v)
-	return offset + 1
+	return base
 }
 func (m *DatabaseRecord) Size() (n int) {
+	if m == nil {
+		return 0
+	}
 	var l int
 	_ = l
 	if len(m.Addresses) > 0 {
@@ -221,10 +334,16 @@ func (m *DatabaseRecord) Size() (n int) {
 	if m.Seen != 0 {
 		n += 1 + sovDatabase(uint64(m.Seen))
 	}
+	if m.Missed != 0 {
+		n += 1 + sovDatabase(uint64(m.Missed))
+	}
 	return n
 }
 
 func (m *ReplicationRecord) Size() (n int) {
+	if m == nil {
+		return 0
+	}
 	var l int
 	_ = l
 	l = len(m.Key)
@@ -244,6 +363,9 @@ func (m *ReplicationRecord) Size() (n int) {
 }
 
 func (m *DatabaseAddress) Size() (n int) {
+	if m == nil {
+		return 0
+	}
 	var l int
 	_ = l
 	l = len(m.Address)
@@ -257,14 +379,7 @@ func (m *DatabaseAddress) Size() (n int) {
 }
 
 func sovDatabase(x uint64) (n int) {
-	for {
-		n++
-		x >>= 7
-		if x == 0 {
-			break
-		}
-	}
-	return n
+	return (math_bits.Len64(x|1) + 6) / 7
 }
 func sozDatabase(x uint64) (n int) {
 	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
@@ -284,7 +399,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -312,7 +427,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= (int(b) & 0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -321,6 +436,9 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -343,7 +461,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Misses |= (int32(b) & 0x7F) << shift
+				m.Misses |= int32(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -362,7 +480,26 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= (int64(b) & 0x7F) << shift
+				m.Seen |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Missed", wireType)
+			}
+			m.Missed = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Missed |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -376,6 +513,9 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -403,7 +543,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -431,7 +571,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -441,6 +581,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -460,7 +603,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= (int(b) & 0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -469,6 +612,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -491,7 +637,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= (int64(b) & 0x7F) << shift
+				m.Seen |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -505,6 +651,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -532,7 +681,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -560,7 +709,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -570,6 +719,9 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -589,7 +741,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Expires |= (int64(b) & 0x7F) << shift
+				m.Expires |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -603,6 +755,9 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -618,6 +773,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 func skipDatabase(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0
+	depth := 0
 	for iNdEx < l {
 		var wire uint64
 		for shift := uint(0); ; shift += 7 {
@@ -649,10 +805,8 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			return iNdEx, nil
 		case 1:
 			iNdEx += 8
-			return iNdEx, nil
 		case 2:
 			var length int
 			for shift := uint(0); ; shift += 7 {
@@ -669,75 +823,34 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			iNdEx += length
 			if length < 0 {
 				return 0, ErrInvalidLengthDatabase
 			}
-			return iNdEx, nil
+			iNdEx += length
 		case 3:
-			for {
-				var innerWire uint64
-				var start int = iNdEx
-				for shift := uint(0); ; shift += 7 {
-					if shift >= 64 {
-						return 0, ErrIntOverflowDatabase
-					}
-					if iNdEx >= l {
-						return 0, io.ErrUnexpectedEOF
-					}
-					b := dAtA[iNdEx]
-					iNdEx++
-					innerWire |= (uint64(b) & 0x7F) << shift
-					if b < 0x80 {
-						break
-					}
-				}
-				innerWireType := int(innerWire & 0x7)
-				if innerWireType == 4 {
-					break
-				}
-				next, err := skipDatabase(dAtA[start:])
-				if err != nil {
-					return 0, err
-				}
-				iNdEx = start + next
-			}
-			return iNdEx, nil
+			depth++
 		case 4:
-			return iNdEx, nil
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupDatabase
+			}
+			depth--
 		case 5:
 			iNdEx += 4
-			return iNdEx, nil
 		default:
 			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
 		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthDatabase
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
 	}
-	panic("unreachable")
+	return 0, io.ErrUnexpectedEOF
 }
 
 var (
-	ErrInvalidLengthDatabase = fmt.Errorf("proto: negative length found during unmarshaling")
-	ErrIntOverflowDatabase   = fmt.Errorf("proto: integer overflow")
+	ErrInvalidLengthDatabase        = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowDatabase          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupDatabase = fmt.Errorf("proto: unexpected end of group")
 )
-
-func init() { proto.RegisterFile("database.proto", fileDescriptorDatabase) }
-
-var fileDescriptorDatabase = []byte{
-	// 254 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4b, 0x49, 0x2c, 0x49,
-	0x4c, 0x4a, 0x2c, 0x4e, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0xc9, 0x4d, 0xcc, 0xcc,
-	0x93, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf, 0x4f,
-	0xcf, 0xd7, 0x07, 0x4b, 0x26, 0x95, 0xa6, 0x81, 0x79, 0x60, 0x0e, 0x98, 0x05, 0xd1, 0xa4, 0x54,
-	0xce, 0xc5, 0xe7, 0x02, 0x35, 0x26, 0x28, 0x35, 0x39, 0xbf, 0x28, 0x45, 0xc8, 0x92, 0x8b, 0x33,
-	0x31, 0x25, 0xa5, 0x28, 0xb5, 0xb8, 0x38, 0xb5, 0x58, 0x82, 0x51, 0x81, 0x59, 0x83, 0xdb, 0x48,
-	0x54, 0x0f, 0x64, 0xb4, 0x1e, 0x4c, 0xa1, 0x23, 0x44, 0xda, 0x89, 0xe5, 0xc4, 0x3d, 0x79, 0x86,
-	0x20, 0x84, 0x6a, 0x21, 0x31, 0x2e, 0xb6, 0xdc, 0x4c, 0xb0, 0x3e, 0x26, 0x05, 0x46, 0x0d, 0xd6,
-	0x20, 0x28, 0x4f, 0x48, 0x88, 0x8b, 0xa5, 0x38, 0x35, 0x35, 0x4f, 0x82, 0x59, 0x81, 0x51, 0x83,
-	0x39, 0x08, 0xcc, 0x56, 0x2a, 0xe1, 0x12, 0x0c, 0x4a, 0x2d, 0xc8, 0xc9, 0x4c, 0x4e, 0x2c, 0xc9,
-	0xcc, 0xcf, 0x83, 0xda, 0x2d, 0xc0, 0xc5, 0x9c, 0x9d, 0x5a, 0x29, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1,
-	0x19, 0x04, 0x62, 0xa2, 0xba, 0x86, 0x89, 0x24, 0xd7, 0x60, 0xb3, 0xd5, 0x95, 0x8b, 0x1f, 0x4d,
-	0x9f, 0x90, 0x04, 0x17, 0x3b, 0x54, 0x0f, 0xd4, 0x5e, 0x18, 0x17, 0x24, 0x93, 0x5a, 0x51, 0x90,
-	0x59, 0x04, 0xf5, 0x0f, 0x73, 0x10, 0x8c, 0xeb, 0x24, 0x70, 0xe2, 0xa1, 0x1c, 0xc3, 0x89, 0x47,
-	0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x98, 0xc4, 0x06, 0x0e, 0x4e, 0x63,
-	0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0x9e, 0x45, 0x60, 0x7e, 0x95, 0x01, 0x00, 0x00,
-}

cmd/stdiscosrv/database.proto

@@ -8,16 +8,22 @@ syntax = "proto3";
 
 package main;
 
-import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+import "repos/protobuf/gogoproto/gogo.proto";
 
 option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.goproto_unkeyed_all) = false;
+option (gogoproto.goproto_unrecognized_all) = false;
+option (gogoproto.goproto_sizecache_all) = false;
 
 message DatabaseRecord {
     repeated DatabaseAddress addresses = 1 [(gogoproto.nullable) = false];
-    int32                    misses    = 2; // Number of lookups without hits
+    int32                    misses    = 2; // Number of lookups* without hits
     int64                    seen      = 3; // Unix nanos, last device announce
+    int64                    missed    = 4; // Unix nanos, last* failed lookup
 }
 
+// *) Not every lookup results in a write, so may not be completely accurate
+
 message ReplicationRecord {
     string                   key       = 1;
     repeated DatabaseAddress addresses = 2 [(gogoproto.nullable) = false];

cmd/stdiscosrv/database_test.go

@@ -154,7 +154,7 @@ func TestDatabaseGetSet(t *testing.T) {
 	}
 	if len(rec.Addresses) != 1 {
 		t.Log(rec.Addresses)
-		t.Fatal("should have one addres")
+		t.Fatal("should have one address")
 	}
 	if rec.Misses != 0 {
 		t.Log(rec.Misses)

cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv

@@ -0,0 +1,4 @@
+[stdiscosrv]
+title=Syncthing discovery server
+description=Lets syncthing clients discover each other
+ports=8443/tcp

cmd/stdiscosrv/etc/linux-systemd/default

@@ -0,0 +1,3 @@
+# Default settings for syncthing-discosrv (stdiscosrv).
+## Add Options here:
+DISCOSRV_OPTS=

cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=Syncthing Discovery Server
+After=network.target
+Documentation=man:stdiscosrv(1)
+
+[Service]
+WorkingDirectory=/var/lib/syncthing-discosrv
+EnvironmentFile=/etc/default/syncthing-discosrv
+ExecStart=/usr/bin/stdiscosrv $DISCOSRV_OPTS
+
+# Hardening
+User=syncthing-discosrv
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-discosrv
+NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
+
+[Install]
+WantedBy=multi-user.target
+Alias=syncthing-discosrv.service

cmd/stdiscosrv/main.go

@@ -9,17 +9,15 @@ package main
 import (
 	"crypto/tls"
 	"flag"
-	"fmt"
 	"log"
 	"net"
 	"net/http"
 	"os"
-	"runtime"
-	"strconv"
 	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -65,34 +63,11 @@ var levelDBOptions = &opt.Options{
 	WriteBuffer: 32 << 20, // default 4<<20
 }
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`stdiscosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
-}
-
 var (
 	debug = false
 )
 
 func main() {
-	const (
-		cleanIntv = 1 * time.Hour
-		statsIntv = 5 * time.Minute
-	)
-
 	var listen string
 	var dir string
 	var metricsListen string
@@ -114,19 +89,24 @@ func main() {
 	flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
 	flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
 	flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
-	log.Println(LongVersion)
+	log.Println(build.LongVersionFor("stdiscosrv"))
+	if *showVersion {
+		return
+	}
 
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
-	if err != nil {
+	if os.IsNotExist(err) {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 0)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
+	} else if err != nil {
+		log.Fatalln("Failed to load keypair:", err)
 	}
-
 	devID := protocol.NewDeviceID(cert.Certificate[0])
 	log.Println("Server device ID is", devID)
 
@@ -164,7 +144,9 @@ func main() {
 	}
 
 	// Root of the service tree.
-	main := suture.NewSimple("main")
+	main := suture.New("main", suture.Spec{
+		PassThroughPanics: true,
+	})
 
 	// Start the database.
 	db, err := newLevelDBStore(dir)

cmd/stdiscosrv/replication.go

@@ -18,6 +18,9 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
+const replicationReadTimeout = time.Minute
+const replicationHeartbeatInterval = time.Second * 30
+
 type replicator interface {
 	send(key string, addrs []DatabaseAddress, seen int64)
 }
@@ -79,10 +82,22 @@ func (s *replicationSender) Serve() {
 		return
 	}
 
+	heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
+	defer heartBeatTicker.Stop()
+
 	// Send records.
 	buf := make([]byte, 1024)
 	for {
 		select {
+		case <-heartBeatTicker.C:
+			if len(s.outbox) > 0 {
+				// No need to send heartbeats if there are events/prevrious
+				// heartbeats to send, they will keep the connection alive.
+				continue
+			}
+			// Empty replication message is the heartbeat:
+			s.outbox <- ReplicationRecord{}
+
 		case rec := <-s.outbox:
 			// Buffer must hold record plus four bytes for size
 			size := rec.Size()
@@ -106,6 +121,7 @@ func (s *replicationSender) Serve() {
 			if _, err := conn.Write(buf[:4+n]); err != nil {
 				replicationSendsTotal.WithLabelValues("error").Inc()
 				log.Println("Replication write:", err)
+				// Yes, we are loosing the replication event here.
 				return
 			}
 			replicationSendsTotal.WithLabelValues("success").Inc()
@@ -149,7 +165,7 @@ func (m replicationMultiplexer) send(key string, ps []DatabaseAddress, seen int6
 	}
 }
 
-// replicationListener acceptes incoming connections and reads replication
+// replicationListener accepts incoming connections and reads replication
 // items from them. Incoming items are applied to the KV store.
 type replicationListener struct {
 	addr       string
@@ -242,7 +258,7 @@ func (l *replicationListener) handle(conn net.Conn) {
 		default:
 		}
 
-		conn.SetReadDeadline(time.Now().Add(time.Minute))
+		conn.SetReadDeadline(time.Now().Add(replicationReadTimeout))
 
 		// First four bytes are the size
 		if _, err := io.ReadFull(conn, buf[:4]); err != nil {
@@ -256,6 +272,12 @@ func (l *replicationListener) handle(conn net.Conn) {
 		if len(buf) < size {
 			buf = make([]byte, size)
 		}
+
+		if size == 0 {
+			// Heartbeat, ignore
+			continue
+		}
+
 		if _, err := io.ReadFull(conn, buf[:size]); err != nil {
 			log.Println("Replication read record:", err)
 			replicationRecvsTotal.WithLabelValues("error").Inc()

cmd/stdiscosrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-discosrv --ingroup syncthing syncthing-discosrv

cmd/stdiscosrv/stats.go

@@ -7,6 +7,8 @@
 package main
 
 import (
+	"os"
+
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -93,6 +95,7 @@ const (
 	dbOpGet             = "get"
 	dbOpPut             = "put"
 	dbOpMerge           = "merge"
+	dbOpDelete          = "delete"
 	dbResSuccess        = "success"
 	dbResNotFound       = "not_found"
 	dbResError          = "error"
@@ -105,4 +108,16 @@ func init() {
 		replicationSendsTotal, replicationRecvsTotal,
 		databaseKeys, databaseStatisticsSeconds,
 		databaseOperations, databaseOperationSeconds)
+
+	processCollectorOpts := prometheus.ProcessCollectorOpts{
+		Namespace: "syncthing_discovery",
+		PidFn: func() (int, error) {
+			return os.Getpid(), nil
+		},
+	}
+
+	prometheus.MustRegister(
+		prometheus.NewProcessCollector(processCollectorOpts),
+	)
+
 }

cmd/stfileinfo/main.go

@@ -68,8 +68,8 @@ func main() {
 		}
 
 		blockSize := int(fi.Size())
-		if *standardBlocks || blockSize < protocol.BlockSize {
-			blockSize = protocol.BlockSize
+		if *standardBlocks || blockSize < protocol.MinBlockSize {
+			blockSize = protocol.BlockSize(fi.Size())
 		}
 		bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
 		if err != nil {

cmd/stfinddevice/main.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"errors"
 	"flag"
@@ -17,6 +18,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -82,7 +84,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 }
 
 func checkServer(deviceID protocol.DeviceID, server string) checkResult {
-	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger)
 	if err != nil {
 		return checkResult{error: err}
 	}
@@ -94,7 +96,7 @@ func checkServer(deviceID protocol.DeviceID, server string) checkResult {
 	})
 
 	go func() {
-		addresses, err := disco.Lookup(deviceID)
+		addresses, err := disco.Lookup(context.Background(), deviceID)
 		res <- checkResult{addresses: addresses, error: err}
 	}()
 

cmd/stfindignored/main.go

@@ -0,0 +1,44 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Commmand stfindignored lists ignored files under a given folder root.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/syncthing/syncthing/lib/fs"
+	"github.com/syncthing/syncthing/lib/ignore"
+)
+
+func main() {
+	flag.Parse()
+	root := flag.Arg(0)
+	if root == "" {
+		root = "."
+	}
+
+	vfs := fs.NewWalkFilesystem(fs.NewFilesystem(fs.FilesystemTypeBasic, root))
+
+	ign := ignore.New(vfs)
+	if err := ign.Load(".stignore"); err != nil {
+		fmt.Fprintf(os.Stderr, "Fatal: loading ignores: %v\n", err)
+		os.Exit(1)
+	}
+
+	vfs.Walk(".", func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Warning: %s: %v\n", path, err)
+			return fs.SkipDir
+		}
+		if ign.Match(path).IsIgnored() {
+			fmt.Println(path)
+		}
+		return nil
+	})
+}

cmd/stgenfiles/main.go

@@ -82,7 +82,7 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
 		return err
 	}
 
-	_ = os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
+	os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
 
 	t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
 	return os.Chtimes(p1, t, t)

cmd/stindex/accounting.go

@@ -0,0 +1,58 @@
+// Copyright (C) 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"text/tabwriter"
+
+	"github.com/syncthing/syncthing/lib/db/backend"
+)
+
+// account prints key and data size statistics per class
+func account(ldb backend.Backend) {
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var ksizes [256]int
+	var dsizes [256]int
+	var counts [256]int
+	var max [256]int
+
+	for it.Next() {
+		key := it.Key()
+		t := key[0]
+		ds := len(it.Value())
+		ks := len(key)
+		s := ks + ds
+
+		counts[t]++
+		ksizes[t] += ks
+		dsizes[t] += ds
+		if s > max[t] {
+			max[t] = s
+		}
+	}
+
+	tw := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', tabwriter.AlignRight)
+	toti, totds, totks := 0, 0, 0
+	for t := range ksizes {
+		if ksizes[t] > 0 {
+			// yes metric kilobytes 🤘
+			fmt.Fprintf(tw, "0x%02x:\t%d items,\t%d KB keys +\t%d KB data,\t%d B +\t%d B avg,\t%d B max\t\n", t, counts[t], ksizes[t]/1000, dsizes[t]/1000, ksizes[t]/counts[t], dsizes[t]/counts[t], max[t])
+			toti += counts[t]
+			totds += dsizes[t]
+			totks += ksizes[t]
+		}
+	}
+	fmt.Fprintf(tw, "Total\t%d items,\t%d KB keys +\t%d KB data.\t\n", toti, totks/1000, totds/1000)
+	tw.Flush()
+}

cmd/stindex/dump.go

@@ -13,11 +13,15 @@ import (
 	"time"
 
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func dump(ldb *db.Instance) {
-	it := ldb.NewIterator(nil, nil)
+func dump(ldb backend.Backend) {
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {
@@ -48,10 +52,10 @@ func dump(ldb *db.Instance) {
 			fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
 
 		case db.KeyTypeDeviceStatistic:
-			fmt.Printf("[dstat] K:%x V:%x\n", it.Key(), it.Value())
+			fmt.Printf("[dstat] K:%x V:%x\n", key, it.Value())
 
 		case db.KeyTypeFolderStatistic:
-			fmt.Printf("[fstat] K:%x V:%x\n", it.Key(), it.Value())
+			fmt.Printf("[fstat] K:%x V:%x\n", key, it.Value())
 
 		case db.KeyTypeVirtualMtime:
 			folder := binary.BigEndian.Uint32(key[1:])
@@ -63,21 +67,72 @@ func dump(ldb *db.Instance) {
 			fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, real, virt)
 
 		case db.KeyTypeFolderIdx:
-			key := binary.BigEndian.Uint32(it.Key()[1:])
+			key := binary.BigEndian.Uint32(key[1:])
 			fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
 
 		case db.KeyTypeDeviceIdx:
-			key := binary.BigEndian.Uint32(it.Key()[1:])
+			key := binary.BigEndian.Uint32(key[1:])
 			val := it.Value()
-			if len(val) == 0 {
-				fmt.Printf("[deviceidx] K:%d V:<nil>\n", key)
+			device := "<nil>"
+			if len(val) > 0 {
+				dev, err := protocol.DeviceIDFromBytes(val)
+				if err != nil {
+					device = fmt.Sprintf("<invalid %d bytes>", len(val))
+				} else {
+					device = dev.String()
+				}
+			}
+			fmt.Printf("[deviceidx] K:%d V:%s\n", key, device)
+
+		case db.KeyTypeIndexID:
+			device := binary.BigEndian.Uint32(key[1:])
+			folder := binary.BigEndian.Uint32(key[5:])
+			fmt.Printf("[indexid] D:%d F:%d I:%x\n", device, folder, it.Value())
+
+		case db.KeyTypeFolderMeta:
+			folder := binary.BigEndian.Uint32(key[1:])
+			fmt.Printf("[foldermeta] F:%d", folder)
+			var cs db.CountsSet
+			if err := cs.Unmarshal(it.Value()); err != nil {
+				fmt.Printf(" (invalid)\n")
 			} else {
-				dev := protocol.DeviceIDFromBytes(val)
-				fmt.Printf("[deviceidx] K:%d V:%s\n", key, dev)
+				fmt.Printf(" V:%v\n", cs)
+			}
+
+		case db.KeyTypeMiscData:
+			fmt.Printf("[miscdata] K:%q V:%q\n", key[1:], it.Value())
+
+		case db.KeyTypeSequence:
+			folder := binary.BigEndian.Uint32(key[1:])
+			seq := binary.BigEndian.Uint64(key[5:])
+			fmt.Printf("[sequence] F:%d S:%d V:%q\n", folder, seq, it.Value())
+
+		case db.KeyTypeNeed:
+			folder := binary.BigEndian.Uint32(key[1:])
+			file := string(key[5:])
+			fmt.Printf("[need] F:%d V:%q\n", folder, file)
+
+		case db.KeyTypeBlockList:
+			fmt.Printf("[blocklist] H:%x\n", key[1:])
+
+		case db.KeyTypeBlockListMap:
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[5:37]
+			fileName := string(key[37:])
+			fmt.Printf("[blocklistmap] F:%d H:%x N:%s\n", folder, hash, fileName)
+
+		case db.KeyTypeVersion:
+			fmt.Printf("[version] H:%x", key[1:])
+			var v protocol.Vector
+			err := v.Unmarshal(it.Value())
+			if err != nil {
+				fmt.Printf(" (invalid)\n")
+			} else {
+				fmt.Printf(" V:%v\n", v)
 			}
 
 		default:
-			fmt.Printf("[???]\n  %x\n  %x\n", it.Key(), it.Value())
+			fmt.Printf("[??? %d]\n  %x\n  %x\n", key[0], key, it.Value())
 		}
 	}
 }

cmd/stindex/dumpsize.go

@@ -10,8 +10,10 @@ import (
 	"container/heap"
 	"encoding/binary"
 	"fmt"
+	"log"
 
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
 )
 
 type SizedElement struct {
@@ -37,11 +39,14 @@ func (h *ElementHeap) Pop() interface{} {
 	return x
 }
 
-func dumpsize(ldb *db.Instance) {
+func dumpsize(ldb backend.Backend) {
 	h := &ElementHeap{}
 	heap.Init(h)
 
-	it := ldb.NewIterator(nil, nil)
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
 	var ele SizedElement
 	for it.Next() {
 		key := it.Key()

cmd/stindex/idxck.go

@@ -0,0 +1,342 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"log"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+type fileInfoKey struct {
+	folder uint32
+	device uint32
+	name   string
+}
+
+type globalKey struct {
+	folder uint32
+	name   string
+}
+
+type sequenceKey struct {
+	folder   uint32
+	sequence uint64
+}
+
+func idxck(ldb backend.Backend) (success bool) {
+	folders := make(map[uint32]string)
+	devices := make(map[uint32]string)
+	deviceToIDs := make(map[string]uint32)
+	fileInfos := make(map[fileInfoKey]protocol.FileInfo)
+	globals := make(map[globalKey]db.VersionList)
+	sequences := make(map[sequenceKey]string)
+	needs := make(map[globalKey]struct{})
+	blocklists := make(map[string]struct{})
+	versions := make(map[string]protocol.Vector)
+	usedBlocklists := make(map[string]struct{})
+	usedVersions := make(map[string]struct{})
+	var localDeviceKey uint32
+	success = true
+
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+	for it.Next() {
+		key := it.Key()
+		switch key[0] {
+		case db.KeyTypeDevice:
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+
+			var f protocol.FileInfo
+			err := f.Unmarshal(it.Value())
+			if err != nil {
+				fmt.Println("Unable to unmarshal FileInfo:", err)
+				success = false
+				continue
+			}
+
+			fileInfos[fileInfoKey{folder, device, name}] = f
+
+		case db.KeyTypeGlobal:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			var flv db.VersionList
+			if err := flv.Unmarshal(it.Value()); err != nil {
+				fmt.Println("Unable to unmarshal VersionList:", err)
+				success = false
+				continue
+			}
+			globals[globalKey{folder, name}] = flv
+
+		case db.KeyTypeFolderIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			folders[key] = string(it.Value())
+
+		case db.KeyTypeDeviceIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			devices[key] = string(it.Value())
+			deviceToIDs[string(it.Value())] = key
+			if bytes.Equal(it.Value(), protocol.LocalDeviceID[:]) {
+				localDeviceKey = key
+			}
+
+		case db.KeyTypeSequence:
+			folder := binary.BigEndian.Uint32(key[1:])
+			seq := binary.BigEndian.Uint64(key[5:])
+			val := it.Value()
+			sequences[sequenceKey{folder, seq}] = string(val[9:])
+
+		case db.KeyTypeNeed:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			needs[globalKey{folder, name}] = struct{}{}
+
+		case db.KeyTypeBlockList:
+			hash := string(key[1:])
+			blocklists[hash] = struct{}{}
+
+		case db.KeyTypeVersion:
+			hash := string(key[1:])
+			var v protocol.Vector
+			if err := v.Unmarshal(it.Value()); err != nil {
+				fmt.Println("Unable to unmarshal Vector:", err)
+				success = false
+				continue
+			}
+			versions[hash] = v
+		}
+	}
+
+	if localDeviceKey == 0 {
+		fmt.Println("Missing key for local device in device index (bailing out)")
+		success = false
+		return
+	}
+
+	var missingSeq []sequenceKey
+	for fk, fi := range fileInfos {
+		if fk.name != fi.Name {
+			fmt.Printf("Mismatching FileInfo name, %q (key) != %q (actual)\n", fk.name, fi.Name)
+			success = false
+		}
+
+		folder := folders[fk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for FileInfo %q\n", fk.folder, fk.name)
+			success = false
+			continue
+		}
+		if devices[fk.device] == "" {
+			fmt.Printf("Unknown device ID %d for FileInfo %q, folder %q\n", fk.folder, fk.name, folder)
+			success = false
+		}
+
+		if fk.device == localDeviceKey {
+			sk := sequenceKey{fk.folder, uint64(fi.Sequence)}
+			name, ok := sequences[sk]
+			if !ok {
+				fmt.Printf("Sequence entry missing for FileInfo %q, folder %q, seq %d\n", fi.Name, folder, fi.Sequence)
+				missingSeq = append(missingSeq, sk)
+				success = false
+				continue
+			}
+			if name != fi.Name {
+				fmt.Printf("Sequence entry refers to wrong name, %q (seq) != %q (FileInfo), folder %q, seq %d\n", name, fi.Name, folder, fi.Sequence)
+				success = false
+			}
+		}
+
+		if len(fi.Blocks) == 0 && len(fi.BlocksHash) != 0 {
+			key := string(fi.BlocksHash)
+			if _, ok := blocklists[key]; !ok {
+				fmt.Printf("Missing block list for file %q, block list hash %x\n", fi.Name, fi.BlocksHash)
+				success = false
+			} else {
+				usedBlocklists[key] = struct{}{}
+			}
+		}
+
+		if fi.VersionHash != nil {
+			key := string(fi.VersionHash)
+			if _, ok := versions[key]; !ok {
+				fmt.Printf("Missing version vector for file %q, version hash %x\n", fi.Name, fi.VersionHash)
+				success = false
+			} else {
+				usedVersions[key] = struct{}{}
+			}
+		}
+
+		_, ok := globals[globalKey{fk.folder, fk.name}]
+		if !ok {
+			fmt.Printf("Missing global for file %q\n", fi.Name)
+			success = false
+			continue
+		}
+	}
+
+	// Aggregate the ranges of missing sequence entries, print them
+
+	sort.Slice(missingSeq, func(a, b int) bool {
+		if missingSeq[a].folder != missingSeq[b].folder {
+			return missingSeq[a].folder < missingSeq[b].folder
+		}
+		return missingSeq[a].sequence < missingSeq[b].sequence
+	})
+
+	var folder uint32
+	var startSeq, prevSeq uint64
+	for _, sk := range missingSeq {
+		if folder != sk.folder || sk.sequence != prevSeq+1 {
+			if folder != 0 {
+				fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
+			}
+			startSeq = sk.sequence
+			folder = sk.folder
+		}
+		prevSeq = sk.sequence
+	}
+	if folder != 0 {
+		fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
+	}
+
+	for gk, vl := range globals {
+		folder := folders[gk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
+			success = false
+		}
+		checkGlobal := func(i int, device []byte, version protocol.Vector, invalid, deleted bool) {
+			dev, ok := deviceToIDs[string(device)]
+			if !ok {
+				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, device)
+				success = false
+			}
+			fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
+			if !ok {
+				fmt.Printf("VersionList %q, folder %q, entry %d refers to unknown FileInfo\n", gk.name, folder, i)
+				success = false
+			}
+
+			fiv := fi.Version
+			if fi.VersionHash != nil {
+				fiv = versions[string(fi.VersionHash)]
+			}
+			if !fiv.Equal(version) {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, version, fi.Version)
+				success = false
+			}
+			if fi.IsInvalid() != invalid {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, invalid, fi.IsInvalid())
+				success = false
+			}
+			if fi.IsDeleted() != deleted {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo deleted mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, deleted, fi.IsDeleted())
+				success = false
+			}
+		}
+		for i, fv := range vl.RawVersions {
+			for _, device := range fv.Devices {
+				checkGlobal(i, device, fv.Version, false, fv.Deleted)
+			}
+			for _, device := range fv.InvalidDevices {
+				checkGlobal(i, device, fv.Version, true, fv.Deleted)
+			}
+		}
+
+		// If we need this file we should have a need entry for it. False
+		// positives from needsLocally for deleted files, where we might
+		// legitimately lack an entry if we never had it, and ignored files.
+		if needsLocally(vl) {
+			_, ok := needs[gk]
+			if !ok {
+				fv, _ := vl.GetGlobal()
+				devB, _ := fv.FirstDevice()
+				dev := deviceToIDs[string(devB)]
+				fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
+				if !fi.IsDeleted() && !fi.IsIgnored() {
+					fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
+				}
+			}
+		}
+	}
+
+	seenSeq := make(map[fileInfoKey]uint64)
+	for sk, name := range sequences {
+		folder := folders[sk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for sequence entry %d, %q\n", sk.folder, sk.sequence, name)
+			success = false
+			continue
+		}
+
+		if prev, ok := seenSeq[fileInfoKey{folder: sk.folder, name: name}]; ok {
+			fmt.Printf("Duplicate sequence entry for %q, folder %q, seq %d (prev %d)\n", name, folder, sk.sequence, prev)
+			success = false
+		}
+		seenSeq[fileInfoKey{folder: sk.folder, name: name}] = sk.sequence
+
+		fi, ok := fileInfos[fileInfoKey{sk.folder, localDeviceKey, name}]
+		if !ok {
+			fmt.Printf("Missing FileInfo for sequence entry %d, folder %q, %q\n", sk.sequence, folder, name)
+			success = false
+			continue
+		}
+		if fi.Sequence != int64(sk.sequence) {
+			fmt.Printf("Sequence mismatch for %q, folder %q, %d (key) != %d (FileInfo)\n", name, folder, sk.sequence, fi.Sequence)
+			success = false
+		}
+	}
+
+	for nk := range needs {
+		folder := folders[nk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for need entry %q\n", nk.folder, nk.name)
+			success = false
+			continue
+		}
+
+		vl, ok := globals[nk]
+		if !ok {
+			fmt.Printf("Missing global for need entry %q, folder %q\n", nk.name, folder)
+			success = false
+			continue
+		}
+
+		if !needsLocally(vl) {
+			fmt.Printf("Need entry for file we don't need, %q, folder %q\n", nk.name, folder)
+			success = false
+		}
+	}
+
+	if d := len(blocklists) - len(usedBlocklists); d > 0 {
+		fmt.Printf("%d block list entries out of %d needs GC\n", d, len(blocklists))
+	}
+	if d := len(versions) - len(usedVersions); d > 0 {
+		fmt.Printf("%d version entries out of %d needs GC\n", d, len(versions))
+	}
+
+	return
+}
+
+func needsLocally(vl db.VersionList) bool {
+	gfv, gok := vl.GetGlobal()
+	if !gok { // That's weird, but we hardly need something non-existant
+		return false
+	}
+	fv, ok := vl.Get(protocol.LocalDeviceID[:])
+	return db.Need(gfv, ok, fv.Version)
+}

cmd/stindex/main.go

@@ -13,7 +13,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
 )
 
 func main() {
@@ -21,7 +21,7 @@ func main() {
 	log.SetFlags(0)
 	log.SetOutput(os.Stdout)
 
-	flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize")
+	flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize, idxck")
 
 	flag.Parse()
 
@@ -30,18 +30,34 @@ func main() {
 		path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
 	}
 
-	fmt.Println("Path:", path)
-
-	ldb, err := db.Open(path)
+	var ldb backend.Backend
+	var err error
+	if looksLikeBadger(path) {
+		ldb, err = backend.OpenBadger(path)
+	} else {
+		ldb, err = backend.OpenLevelDBRO(path)
+	}
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	if mode == "dump" {
+	switch mode {
+	case "dump":
 		dump(ldb)
-	} else if mode == "dumpsize" {
+	case "dumpsize":
 		dumpsize(ldb)
-	} else {
+	case "idxck":
+		if !idxck(ldb) {
+			os.Exit(1)
+		}
+	case "account":
+		account(ldb)
+	default:
 		fmt.Println("Unknown mode")
 	}
 }
+
+func looksLikeBadger(path string) bool {
+	_, err := os.Stat(filepath.Join(path, "KEYREGISTRY"))
+	return err == nil
+}

cmd/strelaypoolsrv/auto/.gitignore

@@ -1 +1 @@
-gui.go
+gui.files.go

cmd/strelaypoolsrv/auto/doc.go

@@ -0,0 +1,10 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//go:generate go run ../../../script/genassets.go -o gui.files.go ../gui
+
+// Package auto contains auto generated files for web assets.
+package auto

cmd/strelaypoolsrv/gui/index.html

@@ -9,8 +9,15 @@
     <meta name="author" content=""/>
 
     <title>Relay stats</title>
-    <link href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
-    <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/font-awesome/4.6.1/css/font-awesome.min.css"/>
+    <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"/>
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
+    <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"/>
+    <link rel="stylesheet" href="https://unpkg.com/leaflet@1.6.0/dist/leaflet.css"
+  integrity="sha512-xwE/Az9zrjBIphAcBb3F6JVqxf46+CDLwfLMHloNu6KEQCAWi6HcDUbeOfBIptF7tcCzusKFjFw2yuvEpDL9wQ=="
+  crossorigin=""/>
+  <script src="https://unpkg.com/leaflet@1.6.0/dist/leaflet.js"
+  integrity="sha512-gZwIG9x3wUXg2hdXF6+rVkLF/0Vi9U8D2Ntg4Ga5I5BZpVkVxlJWbSQtXPSiUTtC0TjtGOmxa1AJPuV0CPthew=="
+  crossorigin=""></script>
 
     <style>
       #map {
@@ -38,13 +45,15 @@
     <div class="container">
       <h1>Relay Pool Data</h1>
       <div ng-if="relays === undefined" class="text-center">
-        <img src="//cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
+        <img src="https://cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
         <p>Please wait while we gather data</p>
       </div>
       <div>
         <div ng-show="relays !== undefined" class="ng-hide">
           <p>
-            Currently {{ relays.length }} relays online ({{ totals.goMaxProcs }} cores in total).
+            The relays listed on this page are not managed or vetted by the Syncthing project.
+            Each relay is the responsibility of the relay operator.
+            Currently {{ relays.length }} relays online.
           </p>
         </div>
         <div id="map"></div> <!-- Can't hide the map, otherwise it freaks out -->
@@ -56,83 +65,83 @@
             <tr>
               <th rowspan="2">Address</td>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.numActiveSessions'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.numActiveSessions'; sortReverse = !sortReverse">
                   Sessions
-                  <span ng-show="sortType == 'status.numActiveSessions' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.numActiveSessions' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.numActiveSessions' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.numActiveSessions' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.numConnections'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.numConnections'; sortReverse = !sortReverse">
                   Connections
-                  <span ng-show="sortType == 'status.numConnections' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.numConnections' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.numConnections' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.numConnections' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.bytesProxied'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.bytesProxied'; sortReverse = !sortReverse">
                   Data relayed
-                  <span ng-show="sortType == 'status.bytesProxied' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.bytesProxied' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.bytesProxied' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.bytesProxied' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th colspan="6" class="text-center">Transfer rate in the last period</th>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.uptimeSeconds'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.uptimeSeconds'; sortReverse = !sortReverse">
                   Uptime hours
-                  <span ng-show="sortType == 'status.uptimeSeconds' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.uptimeSeconds' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.uptimeSeconds' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.uptimeSeconds' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.options[\'provided-by\'] || \'\''; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.options[\'provided-by\'] || \'\''; sortReverse = !sortReverse">
                   Provided by
-                  <span ng-show="sortType == 'status.options[\'provided-by\'] || \'\'' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.options[\'provided-by\'] || \'\'' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.options[\'provided-by\'] || \'\'' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.options[\'provided-by\'] || \'\'' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
             </tr>
             <tr>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[0]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[0]'; sortReverse = !sortReverse">
                   10s
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[0]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[0]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[0]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[0]' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[1]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[1]'; sortReverse = !sortReverse">
                   1m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[1]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[1]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[1]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[1]' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[2]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[2]'; sortReverse = !sortReverse">
                   5m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[2]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[2]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[2]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[2]' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[3]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[3]'; sortReverse = !sortReverse">
                   15m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[3]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[3]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[3]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[3]' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[4]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[4]'; sortReverse = !sortReverse">
                   30m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[4]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[4]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[4]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[4]' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[5]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[5]'; sortReverse = !sortReverse">
                   60m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[5]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[5]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[5]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[5]' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
             </tr>
@@ -140,21 +149,21 @@
           <tbody>
             <tr ng-repeat="relay in relays | orderBy:sortType:sortReverse:sortCompare" ng-mouseover="relay.showMarker()" ng-mouseleave="relay.hideMarker()">
               <td>{{ relay.address }}</td>
-              <td ng-if="relay.status === undefined" colspan="11" class="text-center">Looking up...</td>
-              <td ng-if-start="relay.status !== undefined">{{ relay.status.numActiveSessions }}</td>
-              <td>{{ relay.status.numConnections }}</td>
-              <td>{{ relay.status.bytesProxied | bytes }}</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
-              <td ng-if="relay.status.uptimeSeconds != undefined">{{ relay.status.uptimeSeconds/60/60 | number:0 }}</td>
-              <td ng-if="relay.status.uptimeSeconds == undefined"></td>
-              <td title="{{ relay.status.options['provided-by'] || '' }}" ng-if-end>
-                {{ relay.status.options['provided-by'] || '' | limitTo:50 }}
-                <span ng-if="(relay.status.options['provided-by'] || '').length > 50">&hellip;
+              <td ng-if="!relay.stats" colspan="11"></td>
+              <td ng-if-start="relay.stats">{{ relay.stats.numActiveSessions }}</td>
+              <td>{{ relay.stats.numConnections }}</td>
+              <td>{{ relay.stats.bytesProxied | bytes }}</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
+              <td ng-if="relay.stats.uptimeSeconds != undefined">{{ relay.stats.uptimeSeconds/60/60 | number:0 }}</td>
+              <td ng-if="relay.stats.uptimeSeconds == undefined"></td>
+              <td title="{{ relay.stats.options['provided-by'] || '' }}" ng-if-end>
+                {{ relay.stats.options['provided-by'] || '' | limitTo:50 }}
+                <span ng-if="(relay.stats.options['provided-by'] || '').length > 50">&hellip;
               </td>
             </tr>
           </tbody>
@@ -184,18 +193,17 @@
     </div>
 
 
-    <script type="text/javascript" src="//code.jquery.com/jquery-2.1.4.min.js"></script>
-    <script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
-    <script type="text/javascript" src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
-    <script type="text/javascript" src="//maps.googleapis.com/maps/api/js"></script>
+    <script type="text/javascript" src="https://code.jquery.com/jquery-2.1.4.min.js"></script>
+    <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
+    <script type="text/javascript" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
   </body>
 
   <script>
   angular.module('syncthing', [
   ])
-  .config(function($httpProvider) {
+  .config(['$httpProvider', function($httpProvider) {
     $httpProvider.defaults.timeout = 5000;
-  })
+  }])
   .filter('bytes', function() {
     return function(bytes, precision) {
       if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) return '-';
@@ -228,23 +236,24 @@
       numProxies: 0,
       uptimeSeconds: 0,
     };
-    $scope.map = new google.maps.Map(document.getElementById('map'), {
-      zoom: 1,
-      mapTypeId: google.maps.MapTypeId.ROADMAP
-    });
-    $scope.mapBounds = new google.maps.LatLngBounds();
+    $scope.map = L.map('map').setView([40.90296, 1.90925], 2);
+    L.tileLayer('http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',
+    {
+      attribution: 'Leaflet',
+      maxZoom: 17
+    }).addTo($scope.map);
     $scope.tooltipTemplate = $('#infoTemplate').html();
     $scope.usedLocations = {};
-    $scope.sortType = 'status.numActiveSessions';
+    $scope.sortType = 'stats.numActiveSessions';
     $scope.sortReverse = true;
     $scope.sortCompare = function(a, b) {
       if (a.value == b.value) {
         return 0;
       }
-      if (a.type == "undefined") {
+      if (a.type == "undefined" || a.type == "null") {
         return -1;
       }
-      if (b.type == "undefined") {
+      if (b.type == "undefined" || b.type == "null") {
         return 1;
       }
       return a.value > b.value ? 1 : -1;
@@ -252,29 +261,36 @@
 
     $http.get("/endpoint").then(function(response) {
       $scope.relays = response.data.relays;
-      var promises = [];
-      angular.forEach($scope.relays, function(relay) {
 
+      angular.forEach($scope.relays, function(relay) {
         relay.uri = constructURI(relay.url);
         relay.address = relay.url.split('/')[2];
 
         addMarkerToMap(relay);
 
-        promises.push(getRelayStatus(relay));
+        if (relay.stats) {
+          angular.forEach($scope.totals, function(value, key) {
+            if (typeof $scope.totals[key] == 'number') {
+              $scope.totals[key] += relay.stats[key];
+            } else if (typeof $scope.totals[key] == 'object' && $scope.totals[key] instanceof Array) {
+              angular.forEach($scope.totals[key], function(value, index) {
+                $scope.totals[key][index] += relay.stats[key][index];
+              });
+            }
+          });
+        }
       });
 
-      // Can only add circles once we know the totals for transfers, which means
-      // we need to resolve all statuses.
-      $q.all(promises).then(function() {
-        angular.forEach($scope.relays, function(relay) {
-          if (relay.status) {
-            addCircleToMap(relay);
-          }
-        });
+      // After the totals were calculated, add circles.
+      angular.forEach($scope.relays, function(relay) {
+        if (relay.stats) {
+          addCircleToMap(relay);
+        }
       });
 
-      $scope.map.fitBounds($scope.mapBounds);
       if ($scope.relays.length == 1) {
+        //Center to only relay with zoom
+        $scope.map.panTo(new L.LatLng(relays[0].location.latitude, relays[0].location.longitude));
         $scope.map.setZoom(13);
       }
     });
@@ -294,75 +310,50 @@
 
         var locParts = loc.split(',');
 
-        relay.marker = new google.maps.Marker({
-          map: $scope.map,
-          position: new google.maps.LatLng(locParts[0], locParts[1]),
+        relay.marker = new L.Marker([relay.location.latitude, relay.location.longitude],{
           title: relay.url,
         });
 
         var scope = $rootScope.$new(true);
         scope.relay = relay;
 
-        relay.marker.info = new google.maps.InfoWindow({
-          content: $compile($scope.tooltipTemplate)(scope)[0],
+        var icon = new L.Icon({
+          iconSize:     [18, 28], // size of the icon
+          iconAnchor:   [9, 28], // point of the icon which will correspond to marker's location
+          shadowAnchor: [0, 0],  // the same for the shadow
+          popupAnchor:  [0, -27], // popup anchor
+          shadowSize:   [0,0],
+          iconUrl: 'https://cdn.rawgit.com/pointhi/leaflet-color-markers/master/img/marker-icon-red.png',
+          shadowUrl: 'https://cdnjs.cloudflare.com/ajax/libs/leaflet/0.7.7/images/marker-shadow.png',
         });
 
+        relay.marker = new L.marker(new L.latLng(locParts[0], locParts[1]),{icon})
+                            .bindPopup($compile($scope.tooltipTemplate)(scope)[0],{})
+                            .on('mouseover', function (e) {
+                                this.openPopup();
+                            }).on('mouseout', function (e) {
+                                this.closePopup();
+                            }).addTo($scope.map);
+
         relay.showMarker = function() {
-          relay.marker.info.open($scope.map, relay.marker);
+          relay.marker.openPopup();
         }
-
+        
         relay.hideMarker = function() {
-          relay.marker.info.close();
+          relay.marker.closePopup();
         }
+      }
 
-        relay.marker.addListener('mouseover', relay.showMarker);
-        relay.marker.addListener('mouseout', relay.hideMarker);
-
-        $scope.mapBounds.extend(relay.marker.position);
-    }
 
     function addCircleToMap(relay) {
-      relay.marker.circle = new google.maps.Circle({
-        strokeColor: '#FF0000',
-        strokeOpacity: 0.8,
-        strokeWeight: 2,
-        fillColor: '#FF0000',
-        fillOpacity: 0.35,
-        map: $scope.map,
-        center: relay.marker.position,
-        radius: ((relay.status.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000
-      });
-    }
-
-    function getRelayStatus(relay) {
-      // Normal timeout doesn't deal with relays which accept the TCP connection
-      // but don't respond (some firewalls do that), so deal with it this way.
-      var timeoutRequest = $q.defer();
-      var resolveStatus = $q.defer();
-
-
-      $http.get("http://" + relay.uri.hostname + ':' + ((relay.uri.args.statusAddr && relay.uri.args.statusAddr.split(':')[1]) || "22070") + "/status", { timeout: timeoutRequest.promise }).then(function (response) {
-        relay.status = response.data;
-        resolveStatus.resolve();
-        angular.forEach($scope.totals, function(value, key) {
-          if (typeof $scope.totals[key] == 'number') {
-            $scope.totals[key] += response.data[key];
-          } else if (typeof $scope.totals[key] == 'object' && $scope.totals[key] instanceof Array) {
-            angular.forEach($scope.totals[key], function(value, index) {
-              $scope.totals[key][index] += response.data[key][index];
-            });
-          }
-        });
-      }, function() {
-        relay.status = null;
-        resolveStatus.resolve();
-      });
-
-      $timeout(function() {
-        timeoutRequest.resolve();
-      }, 5000);
-
-      return resolveStatus.promise;
+      console.log(relay.location.latitude)
+      L.circle([relay.location.latitude, relay.location.longitude], 
+        {
+          radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000,
+          color: "FF0000",
+          fillColor: "#FF0000",
+          fillOpacity: 0.35,
+        }).addTo($scope.map);
     }
 
     function constructURI(url) {
@@ -385,25 +376,25 @@
 
   <script type="text/template" id="infoTemplate">
     <div>
-      <p><b>{{ relay.uri.hostname }}</b> <span ng-if="relay.status.options['provided-by']">provided by <u>{{ relay.status.options['provided-by'] }}</u></span></p>
-      <div ng-if="relay.status">
-        <span ng-if="relay.status.startTime">Start time: {{ relay.status.startTime | date:"medium" }}</br></span>
-        <span ng-if="relay.status.bytesProxied != undefined">Proxied: {{ relay.status.bytesProxied | bytes }}</br></span>
-        <span ng-if="relay.status.numActiveSessions != undefined">Sessions: {{ relay.status.numActiveSessions }}</br></span>
-        <span ng-if="relay.status.numConnections != undefined">Clients: {{ relay.status.numConnections }}</br></span>
-        <span ng-if="relay.status.options.pools">Pools: {{ relay.status.options.pools.join(', ') }}</br></span>
-        <span ng-if="relay.status.options['global-rate'] != undefined">
-          <span ng-if="relay.status.options['global-rate'] > 0">Global rate limit: {{ relay.status.options['global-rate'] | bytes }}/s</span>
-          <span ng-if="relay.status.options['global-rate'] == 0">Global rate limit: unlimited</span>
+      <p><b>{{ relay.uri.hostname }}</b> <span ng-if="relay.stats.options['provided-by']">provided by <u>{{ relay.stats.options['provided-by'] }}</u></span></p>
+      <div ng-if="relay.stats">
+        <span ng-if="relay.stats.startTime">Start time: {{ relay.stats.startTime | date:"medium" }}</br></span>
+        <span ng-if="relay.stats.bytesProxied != undefined">Proxied: {{ relay.stats.bytesProxied | bytes }}</br></span>
+        <span ng-if="relay.stats.numActiveSessions != undefined">Sessions: {{ relay.stats.numActiveSessions }}</br></span>
+        <span ng-if="relay.stats.numConnections != undefined">Clients: {{ relay.stats.numConnections }}</br></span>
+        <span ng-if="relay.stats.options.pools">Pools: {{ relay.stats.options.pools.join(', ') }}</br></span>
+        <span ng-if="relay.stats.options['global-rate'] != undefined">
+          <span ng-if="relay.stats.options['global-rate'] > 0">Global rate limit: {{ relay.stats.options['global-rate'] | bytes }}/s</span>
+          <span ng-if="relay.stats.options['global-rate'] == 0">Global rate limit: unlimited</span>
           <br/>
         </span>
-        <span ng-if="relay.status.options['per-session-rate'] != undefined">
-          <span ng-if="relay.status.options['per-session-rate'] > 0">Session rate limit: {{ relay.status.options['per-session-rate'] | bytes }}/s</span>
-          <span ng-if="relay.status.options['per-session-rate'] == 0">Session rate limit: unlimited</span>
+        <span ng-if="relay.stats.options['per-session-rate'] != undefined">
+          <span ng-if="relay.stats.options['per-session-rate'] > 0">Session rate limit: {{ relay.stats.options['per-session-rate'] | bytes }}/s</span>
+          <span ng-if="relay.stats.options['per-session-rate'] == 0">Session rate limit: unlimited</span>
           <br/>
         </span>
       </div>
-      <div ng-if="!relay.status">
+      <div ng-if="!relay.stats">
         Data unavailable.
       <div>
     </div>

cmd/strelaypoolsrv/main.go

@@ -1,30 +1,33 @@
 // Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
 
-//go:generate go run ../../script/genassets.go gui >auto/gui.go
-
 package main
 
 import (
-	"bytes"
 	"compress/gzip"
+	"context"
 	"crypto/tls"
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
-	"math/rand"
-	"mime"
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/golang/groupcache/lru"
 	"github.com/oschwald/geoip2-golang"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
+	"github.com/syncthing/syncthing/lib/assets"
+	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/relay/client"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -34,12 +37,42 @@ import (
 type location struct {
 	Latitude  float64 `json:"latitude"`
 	Longitude float64 `json:"longitude"`
+	City      string  `json:"city"`
+	Country   string  `json:"country"`
+	Continent string  `json:"continent"`
 }
 
 type relay struct {
-	URL      string   `json:"url"`
-	Location location `json:"location"`
-	uri      *url.URL
+	URL            string   `json:"url"`
+	Location       location `json:"location"`
+	uri            *url.URL
+	Stats          *stats    `json:"stats"`
+	StatsRetrieved time.Time `json:"statsRetrieved"`
+}
+
+type stats struct {
+	StartTime          time.Time `json:"startTime"`
+	UptimeSeconds      int       `json:"uptimeSeconds"`
+	PendingSessionKeys int       `json:"numPendingSessionKeys"`
+	ActiveSessions     int       `json:"numActiveSessions"`
+	Connections        int       `json:"numConnections"`
+	Proxies            int       `json:"numProxies"`
+	BytesProxied       int       `json:"bytesProxied"`
+	GoVersion          string    `json:"goVersion"`
+	GoOS               string    `json:"goOS"`
+	GoArch             string    `json:"goArch"`
+	GoMaxProcs         int       `json:"goMaxProcs"`
+	GoRoutines         int       `json:"goNumRoutine"`
+	Rates              []int64   `json:"kbps10s1m5m15m30m60m"`
+	Options            struct {
+		NetworkTimeout int      `json:"network-timeout"`
+		PintInterval   int      `json:"ping-interval"`
+		MessageTimeout int      `json:"message-timeout"`
+		SessionRate    int      `json:"per-session-rate"`
+		GlobalRate     int      `json:"global-rate"`
+		Pools          []string `json:"pools"`
+		ProvidedBy     string   `json:"provided-by"`
+	} `json:"options"`
 }
 
 func (r relay) String() string {
@@ -47,9 +80,9 @@ func (r relay) String() string {
 }
 
 type request struct {
-	relay  relay
-	uri    *url.URL
-	result chan result
+	relay      *relay
+	result     chan result
+	queueTimer *prometheus.Timer
 }
 
 type result struct {
@@ -58,56 +91,72 @@ type result struct {
 }
 
 var (
-	testCert       tls.Certificate
-	listen         = ":80"
-	dir            string
-	evictionTime   = time.Hour
-	debug          bool
-	getLRUSize     = 10 << 10
-	getLimitBurst  = 10
-	getLimitAvg    = 1
-	postLRUSize    = 1 << 10
-	postLimitBurst = 2
-	postLimitAvg   = 1
-	getLimit       time.Duration
-	postLimit      time.Duration
-	permRelaysFile string
-	ipHeader       string
-	geoipPath      string
-	proto          string
+	testCert          tls.Certificate
+	knownRelaysFile   = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
+	listen            = ":80"
+	dir               string
+	evictionTime      = time.Hour
+	debug             bool
+	getLRUSize        = 10 << 10
+	getLimitBurst     = 10
+	getLimitAvg       = 2
+	postLRUSize       = 1 << 10
+	postLimitBurst    = 2
+	postLimitAvg      = 2
+	getLimit          time.Duration
+	postLimit         time.Duration
+	permRelaysFile    string
+	ipHeader          string
+	geoipPath         string
+	proto             string
+	statsRefresh      = time.Minute / 2
+	requestQueueLen   = 10
+	requestProcessors = 1
 
-	getMut      = sync.NewRWMutex()
+	getMut      = sync.NewMutex()
 	getLRUCache *lru.Cache
 
-	postMut      = sync.NewRWMutex()
+	postMut      = sync.NewMutex()
 	postLRUCache *lru.Cache
 
-	requests = make(chan request, 10)
+	requests chan request
 
 	mut             = sync.NewRWMutex()
-	knownRelays     = make([]relay, 0)
-	permanentRelays = make([]relay, 0)
+	knownRelays     = make([]*relay, 0)
+	permanentRelays = make([]*relay, 0)
 	evictionTimers  = make(map[string]*time.Timer)
 )
 
+const (
+	httpStatusEnhanceYourCalm = 429
+)
+
 func main() {
+	log.SetOutput(os.Stdout)
+	log.SetFlags(log.Lshortfile)
+
 	flag.StringVar(&listen, "listen", listen, "Listen address")
 	flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
 	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
 	flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
 	flag.IntVar(&getLRUSize, "get-limit-cache", getLRUSize, "Get request limiter cache size")
-	flag.IntVar(&getLimitAvg, "get-limit-avg", 2, "Allowed average get request rate, per 10 s")
+	flag.IntVar(&getLimitAvg, "get-limit-avg", getLimitAvg, "Allowed average get request rate, per 10 s")
 	flag.IntVar(&getLimitBurst, "get-limit-burst", getLimitBurst, "Allowed burst get requests")
 	flag.IntVar(&postLRUSize, "post-limit-cache", postLRUSize, "Post request limiter cache size")
-	flag.IntVar(&postLimitAvg, "post-limit-avg", 2, "Allowed average post request rate, per minute")
+	flag.IntVar(&postLimitAvg, "post-limit-avg", postLimitAvg, "Allowed average post request rate, per minute")
 	flag.IntVar(&postLimitBurst, "post-limit-burst", postLimitBurst, "Allowed burst post requests")
 	flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
 	flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
 	flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
 	flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
+	flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
+	flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
+	flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
 
 	flag.Parse()
 
+	requests = make(chan request, requestQueueLen)
+
 	getLimit = 10 * time.Second / time.Duration(getLimitAvg)
 	postLimit = time.Minute / time.Duration(postLimitAvg)
 
@@ -118,12 +167,32 @@ func main() {
 	var err error
 
 	if permRelaysFile != "" {
-		loadPermanentRelays(permRelaysFile)
+		permanentRelays = loadRelays(permRelaysFile)
 	}
 
 	testCert = createTestCertificate()
 
-	go requestProcessor()
+	for i := 0; i < requestProcessors; i++ {
+		go requestProcessor()
+	}
+
+	// Load relays from cache in the background.
+	// Load them in a serial fashion to make sure any genuine requests
+	// are not dropped.
+	go func() {
+		for _, relay := range loadRelays(knownRelaysFile) {
+			resultChan := make(chan result)
+			requests <- request{relay, resultChan, nil}
+			result := <-resultChan
+			if result.err != nil {
+				relayTestsTotal.WithLabelValues("failed").Inc()
+			} else {
+				relayTestsTotal.WithLabelValues("success").Inc()
+			}
+		}
+		// Run the the stats refresher once the relays are loaded.
+		statsRefresher(statsRefresh)
+	}()
 
 	if dir != "" {
 		if debug {
@@ -169,6 +238,7 @@ func main() {
 	handler := http.NewServeMux()
 	handler.HandleFunc("/", handleAssets)
 	handler.HandleFunc("/endpoint", handleRequest)
+	handler.HandleFunc("/metrics", handleMetrics)
 
 	srv := http.Server{
 		Handler:     handler,
@@ -181,66 +251,42 @@ func main() {
 	}
 }
 
+func handleMetrics(w http.ResponseWriter, r *http.Request) {
+	timer := prometheus.NewTimer(metricsRequestsSeconds)
+	// Acquire the mutex just to make sure we're not caught mid-way stats collection
+	mut.RLock()
+	promhttp.Handler().ServeHTTP(w, r)
+	mut.RUnlock()
+	timer.ObserveDuration()
+}
+
 func handleAssets(w http.ResponseWriter, r *http.Request) {
-	assets := auto.Assets()
+	w.Header().Set("Cache-Control", "no-cache, must-revalidate")
+
 	path := r.URL.Path[1:]
 	if path == "" {
 		path = "index.html"
 	}
 
-	bs, ok := assets[path]
+	as, ok := auto.Assets()[path]
 	if !ok {
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
 
-	mtype := mimeTypeForFile(path)
-	if len(mtype) != 0 {
-		w.Header().Set("Content-Type", mtype)
-	}
-
-	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
-		w.Header().Set("Content-Encoding", "gzip")
-	} else {
-		// ungzip if browser not send gzip accepted header
-		var gr *gzip.Reader
-		gr, _ = gzip.NewReader(bytes.NewReader(bs))
-		bs, _ = ioutil.ReadAll(gr)
-		gr.Close()
-	}
-	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs)))
-
-	w.Write(bs)
-}
-
-func mimeTypeForFile(file string) string {
-	// We use a built in table of the common types since the system
-	// TypeByExtension might be unreliable. But if we don't know, we delegate
-	// to the system.
-	ext := filepath.Ext(file)
-	switch ext {
-	case ".htm", ".html":
-		return "text/html"
-	case ".css":
-		return "text/css"
-	case ".js":
-		return "application/javascript"
-	case ".json":
-		return "application/json"
-	case ".png":
-		return "image/png"
-	case ".ttf":
-		return "application/x-font-ttf"
-	case ".woff":
-		return "application/x-font-woff"
-	case ".svg":
-		return "image/svg+xml"
-	default:
-		return mime.TypeByExtension(ext)
-	}
+	assets.Serve(w, r, as)
 }
 
 func handleRequest(w http.ResponseWriter, r *http.Request) {
+	timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
+
+	w = NewLoggingResponseWriter(w)
+	defer func() {
+		timer.ObserveDuration()
+		lw := w.(*loggingResponseWriter)
+		apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
+	}()
+
 	if ipHeader != "" {
 		r.RemoteAddr = r.Header.Get(ipHeader)
 	}
@@ -248,13 +294,13 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 	switch r.Method {
 	case "GET":
 		if limit(r.RemoteAddr, getLRUCache, getMut, getLimit, getLimitBurst) {
-			w.WriteHeader(429)
+			w.WriteHeader(httpStatusEnhanceYourCalm)
 			return
 		}
 		handleGetRequest(w, r)
 	case "POST":
 		if limit(r.RemoteAddr, postLRUCache, postMut, postLimit, postLimitBurst) {
-			w.WriteHeader(429)
+			w.WriteHeader(httpStatusEnhanceYourCalm)
 			return
 		}
 		handlePostRequest(w, r)
@@ -266,19 +312,27 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func handleGetRequest(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+func handleGetRequest(rw http.ResponseWriter, r *http.Request) {
+	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
+
 	mut.RLock()
-	relays := append(permanentRelays, knownRelays...)
+	relays := make([]*relay, len(permanentRelays)+len(knownRelays))
+	n := copy(relays, permanentRelays)
+	copy(relays[n:], knownRelays)
 	mut.RUnlock()
 
 	// Shuffle
-	for i := range relays {
-		j := rand.Intn(i + 1)
-		relays[i], relays[j] = relays[j], relays[i]
+	rand.Shuffle(relays)
+
+	w := io.Writer(rw)
+	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+		rw.Header().Set("Content-Encoding", "gzip")
+		gw := gzip.NewWriter(rw)
+		defer gw.Close()
+		w = gw
 	}
 
-	json.NewEncoder(w).Encode(map[string][]relay{
+	_ = json.NewEncoder(w).Encode(map[string][]*relay{
 		"relays": relays,
 	})
 }
@@ -292,7 +346,7 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		if debug {
 			log.Println("Failed to parse payload")
 		}
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
@@ -301,7 +355,7 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		if debug {
 			log.Println("Failed to parse URI", newRelay.URL)
 		}
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
@@ -310,18 +364,14 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		if debug {
 			log.Println("Failed to split URI", newRelay.URL)
 		}
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
 	// Get the IP address of the client
-	rhost, _, err := net.SplitHostPort(r.RemoteAddr)
-	if err != nil {
-		if debug {
-			log.Println("Failed to split remote address", r.RemoteAddr)
-		}
-		http.Error(w, err.Error(), 500)
-		return
+	rhost := r.RemoteAddr
+	if host, _, err := net.SplitHostPort(rhost); err == nil {
+		rhost = host
 	}
 
 	ip := net.ParseIP(host)
@@ -333,18 +383,18 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		if debug {
 			log.Println("IP address advertised does not match client IP address", r.RemoteAddr, uri)
 		}
-		http.Error(w, "IP address does not match client IP", http.StatusUnauthorized)
+		http.Error(w, fmt.Sprintf("IP advertised %s does not match client IP %s", host, rhost), http.StatusUnauthorized)
 		return
 	}
+
 	newRelay.uri = uri
-	newRelay.Location = getLocation(uri.Host)
 
 	for _, current := range permanentRelays {
 		if current.uri.Host == newRelay.uri.Host {
 			if debug {
 				log.Println("Asked to add a relay", newRelay, "which exists in permanent list")
 			}
-			http.Error(w, "Invalid request", 500)
+			http.Error(w, "Invalid request", http.StatusBadRequest)
 			return
 		}
 	}
@@ -352,78 +402,105 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 	reschan := make(chan result)
 
 	select {
-	case requests <- request{newRelay, uri, reschan}:
+	case requests <- request{&newRelay, reschan, prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("queue"))}:
 		result := <-reschan
 		if result.err != nil {
-			http.Error(w, result.err.Error(), 500)
+			relayTestsTotal.WithLabelValues("failed").Inc()
+			http.Error(w, result.err.Error(), http.StatusBadRequest)
 			return
 		}
+		relayTestsTotal.WithLabelValues("success").Inc()
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		json.NewEncoder(w).Encode(map[string]time.Duration{
 			"evictionIn": result.eviction,
 		})
 
 	default:
+		relayTestsTotal.WithLabelValues("dropped").Inc()
 		if debug {
 			log.Println("Dropping request")
 		}
-		w.WriteHeader(429)
+		w.WriteHeader(httpStatusEnhanceYourCalm)
 	}
 }
 
 func requestProcessor() {
 	for request := range requests {
-		if debug {
-			log.Println("Request for", request.relay)
-		}
-		if !client.TestRelay(request.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
-			if debug {
-				log.Println("Test for relay", request.relay, "failed")
-			}
-			request.result <- result{fmt.Errorf("test failed"), 0}
-			continue
+		if request.queueTimer != nil {
+			request.queueTimer.ObserveDuration()
 		}
 
-		mut.Lock()
-		timer, ok := evictionTimers[request.relay.uri.Host]
-		if ok {
-			if debug {
-				log.Println("Stopping existing timer for", request.relay)
-			}
-			timer.Stop()
-		}
-
-		for i, current := range knownRelays {
-			if current.uri.Host == request.relay.uri.Host {
-				if debug {
-					log.Println("Relay", request.relay, "already exists")
-				}
-
-				// Evict the old entry anyway, as configuration might have changed.
-				last := len(knownRelays) - 1
-				knownRelays[i] = knownRelays[last]
-				knownRelays = knownRelays[:last]
-
-				goto found
-			}
-		}
-
-		if debug {
-			log.Println("Adding new relay", request.relay)
-		}
-
-	found:
-
-		knownRelays = append(knownRelays, request.relay)
-
-		evictionTimers[request.relay.uri.Host] = time.AfterFunc(evictionTime, evict(request.relay))
-		mut.Unlock()
-		request.result <- result{nil, evictionTime}
+		timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test"))
+		handleRelayTest(request)
+		timer.ObserveDuration()
 	}
-
 }
 
-func evict(relay relay) func() {
+func handleRelayTest(request request) {
+	if debug {
+		log.Println("Request for", request.relay)
+	}
+	if err := client.TestRelay(context.TODO(), request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3); err != nil {
+		if debug {
+			log.Println("Test for relay", request.relay, "failed:", err)
+		}
+		request.result <- result{err, 0}
+		return
+	}
+
+	stats := fetchStats(request.relay)
+	location := getLocation(request.relay.uri.Host)
+
+	mut.Lock()
+	if stats != nil {
+		updateMetrics(request.relay.uri.Host, *stats, location)
+	}
+	request.relay.Stats = stats
+	request.relay.StatsRetrieved = time.Now()
+	request.relay.Location = location
+
+	timer, ok := evictionTimers[request.relay.uri.Host]
+	if ok {
+		if debug {
+			log.Println("Stopping existing timer for", request.relay)
+		}
+		timer.Stop()
+	}
+
+	for i, current := range knownRelays {
+		if current.uri.Host == request.relay.uri.Host {
+			if debug {
+				log.Println("Relay", request.relay, "already exists")
+			}
+
+			// Evict the old entry anyway, as configuration might have changed.
+			last := len(knownRelays) - 1
+			knownRelays[i] = knownRelays[last]
+			knownRelays = knownRelays[:last]
+
+			goto found
+		}
+	}
+
+	if debug {
+		log.Println("Adding new relay", request.relay)
+	}
+
+found:
+
+	knownRelays = append(knownRelays, request.relay)
+	evictionTimers[request.relay.uri.Host] = time.AfterFunc(evictionTime, evict(request.relay))
+
+	mut.Unlock()
+
+	if err := saveRelays(knownRelaysFile, knownRelays); err != nil {
+		log.Println("Failed to write known relays: " + err.Error())
+	}
+
+	request.result <- result{nil, evictionTime}
+}
+
+func evict(relay *relay) func() {
 	return func() {
 		mut.Lock()
 		defer mut.Unlock()
@@ -438,41 +515,38 @@ func evict(relay relay) func() {
 				last := len(knownRelays) - 1
 				knownRelays[i] = knownRelays[last]
 				knownRelays = knownRelays[:last]
+				deleteMetrics(current.uri.Host)
 			}
 		}
 		delete(evictionTimers, relay.uri.Host)
 	}
 }
 
-func limit(addr string, cache *lru.Cache, lock sync.RWMutex, intv time.Duration, burst int) bool {
-	host, _, err := net.SplitHostPort(addr)
-	if err != nil {
-		return false
+func limit(addr string, cache *lru.Cache, lock sync.Mutex, intv time.Duration, burst int) bool {
+	if host, _, err := net.SplitHostPort(addr); err == nil {
+		addr = host
 	}
 
-	lock.RLock()
-	bkt, ok := cache.Get(host)
-	lock.RUnlock()
-	if ok {
-		bkt := bkt.(*rate.Limiter)
-		if !bkt.Allow() {
-			// Rate limit
-			return true
-		}
-	} else {
-		lock.Lock()
-		cache.Add(host, rate.NewLimiter(rate.Every(intv), burst))
-		lock.Unlock()
+	lock.Lock()
+	v, _ := cache.Get(addr)
+	bkt, ok := v.(*rate.Limiter)
+	if !ok {
+		bkt = rate.NewLimiter(rate.Every(intv), burst)
+		cache.Add(addr, bkt)
 	}
-	return false
+	lock.Unlock()
+
+	return !bkt.Allow()
 }
 
-func loadPermanentRelays(file string) {
+func loadRelays(file string) []*relay {
 	content, err := ioutil.ReadFile(file)
 	if err != nil {
-		log.Fatal(err)
+		log.Println("Failed to load relays: " + err.Error())
+		return nil
 	}
 
+	var relays []*relay
 	for _, line := range strings.Split(string(content), "\n") {
 		if len(line) == 0 {
 			continue
@@ -481,21 +555,30 @@ func loadPermanentRelays(file string) {
 		uri, err := url.Parse(line)
 		if err != nil {
 			if debug {
-				log.Println("Skipping permanent relay", line, "due to parse error", err)
+				log.Println("Skipping relay", line, "due to parse error", err)
 			}
 			continue
 
 		}
 
-		permanentRelays = append(permanentRelays, relay{
+		relays = append(relays, &relay{
 			URL:      line,
 			Location: getLocation(uri.Host),
 			uri:      uri,
 		})
 		if debug {
-			log.Println("Adding permanent relay", line)
+			log.Println("Adding relay", line)
 		}
 	}
+	return relays
+}
+
+func saveRelays(file string, relays []*relay) error {
+	var content string
+	for _, relay := range relays {
+		content += relay.uri.String() + "\n"
+	}
+	return ioutil.WriteFile(file, []byte(content), 0777)
 }
 
 func createTestCertificate() tls.Certificate {
@@ -505,7 +588,7 @@ func createTestCertificate() tls.Certificate {
 	}
 
 	certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
-	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 3072)
+	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 20*365)
 	if err != nil {
 		log.Fatalln("Failed to create test X509 key pair:", err)
 	}
@@ -514,6 +597,8 @@ func createTestCertificate() tls.Certificate {
 }
 
 func getLocation(host string) location {
+	timer := prometheus.NewTimer(locationLookupSeconds)
+	defer timer.ObserveDuration()
 	db, err := geoip2.Open(geoipPath)
 	if err != nil {
 		return location{}
@@ -531,7 +616,24 @@ func getLocation(host string) location {
 	}
 
 	return location{
-		Latitude:  city.Location.Latitude,
 		Longitude: city.Location.Longitude,
+		Latitude:  city.Location.Latitude,
+		City:      city.City.Names["en"],
+		Country:   city.Country.IsoCode,
+		Continent: city.Continent.Code,
 	}
 }
+
+type loggingResponseWriter struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
+	return &loggingResponseWriter{w, http.StatusOK}
+}
+
+func (lrw *loggingResponseWriter) WriteHeader(code int) {
+	lrw.statusCode = code
+	lrw.ResponseWriter.WriteHeader(code)
+}

cmd/strelaypoolsrv/main_test.go

@@ -0,0 +1,67 @@
+// Copyright © 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+)
+
+func init() {
+	for i := 0; i < 10; i++ {
+		u := fmt.Sprintf("permanent%d", i)
+		permanentRelays = append(permanentRelays, &relay{URL: u})
+	}
+
+	knownRelays = []*relay{
+		{URL: "known1"},
+		{URL: "known2"},
+		{URL: "known3"},
+	}
+
+	mut = new(sync.RWMutex)
+}
+
+// Regression test: handleGetRequest should not modify permanentRelays.
+func TestHandleGetRequest(t *testing.T) {
+	needcap := len(permanentRelays) + len(knownRelays)
+	if needcap > cap(permanentRelays) {
+		t.Fatalf("test setup failed: need cap(permanentRelays) >= %d, have %d",
+			needcap, cap(permanentRelays))
+	}
+
+	w := httptest.NewRecorder()
+	w.Body = new(bytes.Buffer)
+	handleGetRequest(w, httptest.NewRequest("GET", "/", nil))
+
+	result := make(map[string][]*relay)
+	err := json.NewDecoder(w.Body).Decode(&result)
+	if err != nil {
+		t.Fatalf("invalid JSON: %v", err)
+	}
+
+	relays := result["relays"]
+	expect, actual := len(knownRelays)+len(permanentRelays), len(relays)
+	if actual != expect {
+		t.Errorf("expected %d relays, got %d", expect, actual)
+	}
+
+	// Check for changes in permanentRelays.
+	for i, r := range permanentRelays {
+		switch {
+		case !strings.HasPrefix(r.URL, "permanent"):
+			t.Errorf("relay %q among permanent relays", r.URL)
+		case r.URL != fmt.Sprintf("permanent%d", i):
+			t.Error("order of permanent relays changed")
+		}
+	}
+}

cmd/strelaypoolsrv/stats.go

@@ -0,0 +1,259 @@
+// Copyright (C) 2018 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"encoding/json"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+func init() {
+	processCollectorOpts := prometheus.ProcessCollectorOpts{
+		Namespace: "syncthing_relaypoolsrv",
+		PidFn: func() (int, error) {
+			return os.Getpid(), nil
+		},
+	}
+
+	prometheus.MustRegister(
+		prometheus.NewProcessCollector(processCollectorOpts),
+	)
+}
+
+var (
+	statusClient = http.Client{
+		Timeout: 5 * time.Second,
+	}
+
+	apiRequestsTotal   = makeCounter("api_requests_total", "Number of API requests.", "type", "result")
+	apiRequestsSeconds = makeSummary("api_requests_seconds", "Latency of API requests.", "type")
+
+	relayTestsTotal         = makeCounter("tests_total", "Number of relay tests.", "result")
+	relayTestActionsSeconds = makeSummary("test_actions_seconds", "Latency of relay test actions.", "type")
+
+	locationLookupSeconds = makeSummary("location_lookup_seconds", "Latency of location lookups.").WithLabelValues()
+
+	metricsRequestsSeconds = makeSummary("metrics_requests_seconds", "Latency of metric requests.").WithLabelValues()
+	scrapeSeconds          = makeSummary("relay_scrape_seconds", "Latency of metric scrapes from remote relays.", "result")
+
+	relayUptime             = makeGauge("relay_uptime", "Uptime of relay", "relay")
+	relayPendingSessionKeys = makeGauge("relay_pending_session_keys", "Number of pending session keys (two keys per session, one per each side of the connection)", "relay")
+	relayActiveSessions     = makeGauge("relay_active_sessions", "Number of sessions that are happening, a session contains two parties", "relay")
+	relayConnections        = makeGauge("relay_connections", "Number of devices connected to the relay", "relay")
+	relayProxies            = makeGauge("relay_proxies", "Number of active proxy routines sending data between peers (two proxies per session, one for each way)", "relay")
+	relayBytesProxied       = makeGauge("relay_bytes_proxied", "Number of bytes proxied by the relay", "relay")
+	relayGoRoutines         = makeGauge("relay_go_routines", "Number of Go routines in the process", "relay")
+	relaySessionRate        = makeGauge("relay_session_rate", "Rate applied per session", "relay")
+	relayGlobalRate         = makeGauge("relay_global_rate", "Global rate applied on the whole relay", "relay")
+	relayBuildInfo          = makeGauge("relay_build_info", "Build information about a relay", "relay", "go_version", "go_os", "go_arch")
+	relayLocationInfo       = makeGauge("relay_location_info", "Location information about a relay", "relay", "city", "country", "continent")
+
+	lastStats = make(map[string]stats)
+)
+
+func makeGauge(name string, help string, labels ...string) *prometheus.GaugeVec {
+	gauge := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "syncthing",
+			Subsystem: "relaypoolsrv",
+			Name:      name,
+			Help:      help,
+		},
+		labels,
+	)
+	prometheus.MustRegister(gauge)
+	return gauge
+}
+
+func makeSummary(name string, help string, labels ...string) *prometheus.SummaryVec {
+	summary := prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Namespace:  "syncthing",
+			Subsystem:  "relaypoolsrv",
+			Name:       name,
+			Help:       help,
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		labels,
+	)
+	prometheus.MustRegister(summary)
+	return summary
+}
+
+func makeCounter(name string, help string, labels ...string) *prometheus.CounterVec {
+	counter := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "relaypoolsrv",
+			Name:      name,
+			Help:      help,
+		},
+		labels,
+	)
+	prometheus.MustRegister(counter)
+	return counter
+}
+
+func statsRefresher(interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	for range ticker.C {
+		refreshStats()
+	}
+}
+
+type statsFetchResult struct {
+	relay *relay
+	stats *stats
+}
+
+func refreshStats() {
+	mut.RLock()
+	relays := append(permanentRelays, knownRelays...)
+	mut.RUnlock()
+
+	now := time.Now()
+	wg := sync.NewWaitGroup()
+
+	results := make(chan statsFetchResult, len(relays))
+	for _, rel := range relays {
+		wg.Add(1)
+		go func(rel *relay) {
+			t0 := time.Now()
+			stats := fetchStats(rel)
+			duration := time.Since(t0).Seconds()
+			result := "success"
+			if stats == nil {
+				result = "failed"
+			}
+			scrapeSeconds.WithLabelValues(result).Observe(duration)
+
+			results <- statsFetchResult{
+				relay: rel,
+				stats: fetchStats(rel),
+			}
+			wg.Done()
+		}(rel)
+	}
+
+	wg.Wait()
+	close(results)
+
+	mut.Lock()
+	relayBuildInfo.Reset()
+	relayLocationInfo.Reset()
+	for result := range results {
+		result.relay.StatsRetrieved = now
+		result.relay.Stats = result.stats
+		if result.stats == nil {
+			deleteMetrics(result.relay.uri.Host)
+		} else {
+			updateMetrics(result.relay.uri.Host, *result.stats, result.relay.Location)
+		}
+	}
+	mut.Unlock()
+}
+
+func fetchStats(relay *relay) *stats {
+	statusAddr := relay.uri.Query().Get("statusAddr")
+	if statusAddr == "" {
+		statusAddr = ":22070"
+	}
+
+	statusHost, statusPort, err := net.SplitHostPort(statusAddr)
+	if err != nil {
+		return nil
+	}
+
+	if statusHost == "" {
+		if host, _, err := net.SplitHostPort(relay.uri.Host); err != nil {
+			return nil
+		} else {
+			statusHost = host
+		}
+	}
+
+	url := "http://" + net.JoinHostPort(statusHost, statusPort) + "/status"
+
+	response, err := statusClient.Get(url)
+	if err != nil {
+		return nil
+	}
+
+	var stats stats
+
+	if json.NewDecoder(response.Body).Decode(&stats); err != nil {
+		return nil
+	}
+	return &stats
+}
+
+func updateMetrics(host string, stats stats, location location) {
+	if stats.GoVersion != "" || stats.GoOS != "" || stats.GoArch != "" {
+		relayBuildInfo.WithLabelValues(host, stats.GoVersion, stats.GoOS, stats.GoArch).Add(1)
+	}
+	if location.City != "" || location.Country != "" || location.Continent != "" {
+		relayLocationInfo.WithLabelValues(host, location.City, location.Country, location.Continent).Add(1)
+	}
+
+	if lastStat, ok := lastStats[host]; ok {
+		stats = mergeStats(stats, lastStat)
+	}
+
+	relayUptime.WithLabelValues(host).Set(float64(stats.UptimeSeconds))
+	relayPendingSessionKeys.WithLabelValues(host).Set(float64(stats.PendingSessionKeys))
+	relayActiveSessions.WithLabelValues(host).Set(float64(stats.ActiveSessions))
+	relayConnections.WithLabelValues(host).Set(float64(stats.Connections))
+	relayProxies.WithLabelValues(host).Set(float64(stats.Proxies))
+	relayBytesProxied.WithLabelValues(host).Set(float64(stats.BytesProxied))
+	relayGoRoutines.WithLabelValues(host).Set(float64(stats.GoRoutines))
+	relaySessionRate.WithLabelValues(host).Set(float64(stats.Options.SessionRate))
+	relayGlobalRate.WithLabelValues(host).Set(float64(stats.Options.GlobalRate))
+	lastStats[host] = stats
+}
+
+func deleteMetrics(host string) {
+	relayUptime.DeleteLabelValues(host)
+	relayPendingSessionKeys.DeleteLabelValues(host)
+	relayActiveSessions.DeleteLabelValues(host)
+	relayConnections.DeleteLabelValues(host)
+	relayProxies.DeleteLabelValues(host)
+	relayBytesProxied.DeleteLabelValues(host)
+	relayGoRoutines.DeleteLabelValues(host)
+	relaySessionRate.DeleteLabelValues(host)
+	relayGlobalRate.DeleteLabelValues(host)
+	delete(lastStats, host)
+}
+
+// Due to some unexplainable behaviour, some of the numbers sometimes travel slightly backwards (by less than 1%)
+// This happens between scrapes, which is 30s, so this can't be a race.
+// This causes prometheus to assume a "rate reset", hence causes phenomenal spikes.
+// One of the number that moves backwards is BytesProxied, which atomically increments a counter with numeric value
+// returned by net.Conn.Read(). I don't think that can return a negative value, so I have no idea what's going on.
+func mergeStats(new stats, old stats) stats {
+	new.UptimeSeconds = mergeValue(new.UptimeSeconds, old.UptimeSeconds)
+	new.PendingSessionKeys = mergeValue(new.PendingSessionKeys, old.PendingSessionKeys)
+	new.ActiveSessions = mergeValue(new.ActiveSessions, old.ActiveSessions)
+	new.Connections = mergeValue(new.Connections, old.Connections)
+	new.Proxies = mergeValue(new.Proxies, old.Proxies)
+	new.BytesProxied = mergeValue(new.BytesProxied, old.BytesProxied)
+	new.GoRoutines = mergeValue(new.GoRoutines, old.GoRoutines)
+	new.Options.SessionRate = mergeValue(new.Options.SessionRate, old.Options.SessionRate)
+	new.Options.GlobalRate = mergeValue(new.Options.GlobalRate, old.Options.GlobalRate)
+	return new
+}
+
+func mergeValue(new, old int) int {
+	if new >= old {
+		return new // normal increase
+	}
+	if float64(new) > 0.99*float64(old) {
+		return old // slight backward movement
+	}
+	return new // reset (relay restart)
+}

cmd/strelaypoolsrv/stats_test.go

@@ -0,0 +1,21 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"testing"
+)
+
+func TestMerge(t *testing.T) {
+	if mergeValue(1001, 1000) != 1001 {
+		t.Error("the computer says no")
+	}
+
+	if mergeValue(999, 1000) != 1000 {
+		t.Error("the computer says no")
+	}
+
+	if mergeValue(1, 1000) != 1 {
+		t.Error("the computer says no")
+	}
+}

cmd/strelaysrv/README.md

@@ -121,11 +121,11 @@ Relay related libraries used by this repo
 ----
 ##### Relay protocol definition.
 
-[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/protocol)
+[Available here](https://github.com/syncthing/syncthing/tree/main/lib/relay/protocol)
 
 
 ##### Relay client
 
 Only used by the testutil.
 
-[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/client)
+[Available here](https://github.com/syncthing/syncthing/tree/main/lib/relay/client)

cmd/strelaysrv/etc/firewall-ufw/strelaysrv

@@ -0,0 +1,9 @@
+[strelaysrv]
+title=Syncthing relay server
+description=Proxies traffic of syncthing client behind firewalls
+ports=22067/tcp
+
+[strelaysrv-metrics]
+title=Syncthing relay metrics
+description=Provides metrics about the syncthing relay server
+ports=22070/tcp

cmd/strelaysrv/etc/linux-systemd/default

@@ -0,0 +1,5 @@
+# Default settings for syncthing-relaysrv (strelaysrv).
+NAT=true
+
+## Add Options here:
+RELAYSRV_OPTS=

cmd/strelaysrv/etc/linux-systemd/strelaysrv.service

@@ -1,17 +1,25 @@
 [Unit]
-Description=Syncthing relay server
+Description=Syncthing Relay Server
 After=network.target
+Documentation=man:strelaysrv(1)
 
 [Service]
-User=strelaysrv
-Group=strelaysrv
-ExecStart=/usr/bin/strelaysrv
-WorkingDirectory=/var/lib/strelaysrv
+WorkingDirectory=/var/lib/syncthing-relaysrv
+EnvironmentFile=/etc/default/syncthing-relaysrv
+ExecStart=/usr/bin/strelaysrv -nat=${NAT} $RELAYSRV_OPTS
 
-PrivateTmp=true
-ProtectSystem=full
-ProtectHome=true
+# Hardening
+User=syncthing-relaysrv
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-relaysrv
 NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
 
 [Install]
 WantedBy=multi-user.target
+Alias=syncthing-relaysrv.service

cmd/strelaysrv/listener.go

@@ -149,7 +149,15 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
 				protocol.WriteMessage(conn, protocol.ResponseSuccess)
 
 			case protocol.ConnectRequest:
-				requestedPeer := syncthingprotocol.DeviceIDFromBytes(msg.ID)
+				requestedPeer, err := syncthingprotocol.DeviceIDFromBytes(msg.ID)
+				if err != nil {
+					if debug {
+						log.Println(id, "is looking for an invalid peer ID")
+					}
+					protocol.WriteMessage(conn, protocol.ResponseNotFound)
+					conn.Close()
+					continue
+				}
 				outboxesMut.RLock()
 				peerOutbox, ok := outboxes[requestedPeer]
 				outboxesMut.RUnlock()

cmd/strelaysrv/main.go

@@ -14,12 +14,13 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
 	"sync/atomic"
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/relay/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -33,24 +34,6 @@ import (
 	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
 )
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`strelaysrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
-}
-
 var (
 	listen string
 	debug  bool
@@ -86,6 +69,12 @@ var (
 	pprofEnabled bool
 )
 
+// httpClient is the HTTP client we use for outbound requests. It has a
+// timeout and may get further options set during initialization.
+var httpClient = &http.Client{
+	Timeout: 30 * time.Second,
+}
+
 func main() {
 	log.SetFlags(log.Lshortfile | log.LstdFlags)
 
@@ -110,8 +99,15 @@ func main() {
 	flag.IntVar(&natTimeout, "nat-timeout", 10, "NAT discovery timeout in seconds")
 	flag.BoolVar(&pprofEnabled, "pprof", false, "Enable the built in profiling on the status server")
 	flag.IntVar(&networkBufferSize, "network-buffer", 2048, "Network buffer size (two of these per proxied connection)")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
+	longVer := build.LongVersionFor("strelaysrv")
+	if *showVersion {
+		fmt.Println(longVer)
+		return
+	}
+
 	if extAddress == "" {
 		extAddress = listen
 	}
@@ -129,18 +125,18 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+
 	if laddr.IP != nil && !laddr.IP.IsUnspecified() {
+		// We bind to a specific address. Our outgoing HTTP requests should
+		// also come from that address.
 		laddr.Port = 0
-		transport, ok := http.DefaultTransport.(*http.Transport)
-		if ok {
-			transport.Dial = (&net.Dialer{
-				Timeout:   30 * time.Second,
-				LocalAddr: laddr,
-			}).Dial
+		boundDialer := &net.Dialer{LocalAddr: laddr}
+		httpClient.Transport = &http.Transport{
+			DialContext: boundDialer.DialContext,
 		}
 	}
 
-	log.Println(LongVersion)
+	log.Println(longVer)
 
 	maxDescriptors, err := osutil.MaximizeOpenFileLimit()
 	if maxDescriptors > 0 {
@@ -160,7 +156,7 @@ func main() {
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 3072)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
@@ -188,7 +184,7 @@ func main() {
 		log.Println("ID:", id)
 	}
 
-	wrapper := config.Wrap("config", config.New(id))
+	wrapper := config.Wrap("config", config.New(id), events.NoopLogger)
 	wrapper.SetOptions(config.OptionsConfiguration{
 		NATLeaseM:   natLease,
 		NATRenewalM: natRenewal,

cmd/strelaysrv/pool.go

@@ -12,6 +12,10 @@ import (
 	"time"
 )
 
+const (
+	httpStatusEnhanceYourCalm = 429
+)
+
 func poolHandler(pool string, uri *url.URL, mapping mapping) {
 	if debug {
 		log.Println("Joining", pool)
@@ -27,40 +31,64 @@ func poolHandler(pool string, uri *url.URL, mapping mapping) {
 			uriCopy.String(),
 		})
 
-		resp, err := http.Post(pool, "application/json", &b)
+		resp, err := httpClient.Post(pool, "application/json", &b)
 		if err != nil {
-			log.Println("Error joining pool", pool, err)
-		} else if resp.StatusCode == 500 {
-			bs, err := ioutil.ReadAll(resp.Body)
-			if err != nil {
-				log.Println("Failed to join", pool, "due to an internal server error. Could not read response:", err)
-			} else {
-				log.Println("Failed to join", pool, "due to an internal server error:", string(bs))
-			}
-			resp.Body.Close()
-		} else if resp.StatusCode == 429 {
-			log.Println(pool, "under load, will retry in a minute")
+			log.Printf("Error joining pool %v: HTTP request: %v", pool, err)
 			time.Sleep(time.Minute)
 			continue
-		} else if resp.StatusCode == 401 {
-			log.Println(pool, "failed to join due to IP address not matching external address. Aborting")
-			return
-		} else if resp.StatusCode == 200 {
+		}
+
+		bs, err := ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+		if err != nil {
+			log.Printf("Error joining pool %v: reading response: %v", pool, err)
+			time.Sleep(time.Minute)
+			continue
+		}
+
+		switch resp.StatusCode {
+		case http.StatusOK:
 			var x struct {
 				EvictionIn time.Duration `json:"evictionIn"`
 			}
-			err := json.NewDecoder(resp.Body).Decode(&x)
-			if err == nil {
+			if err := json.Unmarshal(bs, &x); err == nil {
 				rejoin := x.EvictionIn - (x.EvictionIn / 5)
-				log.Println("Joined", pool, "rejoining in", rejoin)
+				log.Printf("Joined pool %s, rejoining in %v", pool, rejoin)
 				time.Sleep(rejoin)
 				continue
 			} else {
-				log.Println("Failed to deserialize response", err)
+				log.Printf("Joined pool %s, failed to deserialize response: %v", pool, err)
 			}
-		} else {
-			log.Println(pool, "unknown response type from server", resp.StatusCode)
+
+		case http.StatusInternalServerError:
+			log.Printf("Failed to join %v: server error", pool)
+			log.Printf("Response data: %s", bs)
+			time.Sleep(time.Minute)
+			continue
+
+		case http.StatusBadRequest:
+			log.Printf("Failed to join %v: request or check error", pool)
+			log.Printf("Response data: %s", bs)
+			time.Sleep(time.Minute)
+			continue
+
+		case httpStatusEnhanceYourCalm:
+			log.Printf("Failed to join %v: under load (rate limiting)", pool)
+			time.Sleep(time.Minute)
+			continue
+
+		case http.StatusUnauthorized:
+			log.Printf("Failed to join %v: IP address not matching external address", pool)
+			log.Println("Aborting")
+			return
+
+		default:
+			log.Printf("Failed to join %v: unexpected status code from server: %d", pool, resp.StatusCode)
+			log.Printf("Response data: %s", bs)
+			time.Sleep(time.Minute)
+			continue
 		}
+
 		time.Sleep(time.Hour)
 	}
 }

cmd/strelaysrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-relaysrv --ingroup syncthing syncthing-relaysrv

cmd/strelaysrv/session.go

@@ -22,7 +22,7 @@ import (
 var (
 	sessionMut      = sync.RWMutex{}
 	activeSessions  = make([]*session, 0)
-	pendingSessions = make(map[string]*session, 0)
+	pendingSessions = make(map[string]*session)
 	numProxies      int64
 	bytesProxied    int64
 )

cmd/strelaysrv/status.go

@@ -10,6 +10,8 @@ import (
 	"runtime"
 	"sync/atomic"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/build"
 )
 
 var rc *rateCalculator
@@ -40,6 +42,10 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 
 	sessionMut.Lock()
 	// This can potentially be double the number of pending sessions, as each session has two keys, one for each side.
+	status["version"] = build.Version
+	status["buildHost"] = build.Host
+	status["buildUser"] = build.User
+	status["buildDate"] = build.Date
 	status["startTime"] = rc.startTime
 	status["uptimeSeconds"] = time.Since(rc.startTime) / time.Second
 	status["numPendingSessionKeys"] = len(pendingSessions)
@@ -82,9 +88,9 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 }
 
 type rateCalculator struct {
+	counter   *int64 // atomic, must remain 64-bit aligned
 	rates     []int64
 	prev      int64
-	counter   *int64
 	startTime time.Time
 }
 

cmd/strelaysrv/testutil/main.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"crypto/tls"
 	"flag"
 	"log"
@@ -19,6 +20,8 @@ import (
 )
 
 func main() {
+	ctx := context.Background()
+
 	log.SetOutput(os.Stdout)
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
 
@@ -76,7 +79,7 @@ func main() {
 		}()
 
 		for {
-			conn, err := client.JoinSession(<-recv)
+			conn, err := client.JoinSession(ctx, <-recv)
 			if err != nil {
 				log.Fatalln("Failed to join", err)
 			}
@@ -90,13 +93,13 @@ func main() {
 			log.Fatal(err)
 		}
 
-		invite, err := client.GetInvitationFromRelay(uri, id, []tls.Certificate{cert}, 10*time.Second)
+		invite, err := client.GetInvitationFromRelay(ctx, uri, id, []tls.Certificate{cert}, 10*time.Second)
 		if err != nil {
 			log.Fatal(err)
 		}
 
 		log.Println("Received invitation", invite)
-		conn, err := client.JoinSession(invite)
+		conn, err := client.JoinSession(ctx, invite)
 		if err != nil {
 			log.Fatalln("Failed to join", err)
 		}
@@ -104,10 +107,10 @@ func main() {
 		connectToStdio(stdin, conn)
 		log.Println("Finished", conn.RemoteAddr(), conn.LocalAddr())
 	} else if test {
-		if client.TestRelay(uri, []tls.Certificate{cert}, time.Second, 2*time.Second, 4) {
+		if err := client.TestRelay(ctx, uri, []tls.Certificate{cert}, time.Second, 2*time.Second, 4); err == nil {
 			log.Println("OK")
 		} else {
-			log.Println("FAIL")
+			log.Println("FAIL:", err)
 		}
 	} else {
 		log.Fatal("Requires either join or connect")

cmd/stupgrades/main.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"os"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/upgrade"
+)
+
+const defaultURL = "https://api.github.com/repos/syncthing/syncthing/releases?per_page=25"
+
+func main() {
+	url := flag.String("u", defaultURL, "GitHub releases url")
+	flag.Parse()
+
+	rels := upgrade.FetchLatestReleases(*url, "")
+	if rels == nil {
+		// An error was already logged
+		os.Exit(1)
+	}
+
+	sort.Sort(upgrade.SortByRelease(rels))
+	rels = filterForLatest(rels)
+
+	if err := json.NewEncoder(os.Stdout).Encode(rels); err != nil {
+		os.Exit(1)
+	}
+}
+
+// filterForLatest returns the latest stable and prerelease only. If the
+// stable version is newer (comes first in the list) there is no need to go
+// looking for a prerelease at all.
+func filterForLatest(rels []upgrade.Release) []upgrade.Release {
+	var filtered []upgrade.Release
+	var havePre bool
+	for _, rel := range rels {
+		if !rel.Prerelease {
+			// We found a stable version, we're good now.
+			filtered = append(filtered, rel)
+			break
+		}
+		if rel.Prerelease && !havePre {
+			// We remember the first prerelease we find.
+			filtered = append(filtered, rel)
+			havePre = true
+		}
+	}
+	return filtered
+}

cmd/stvanity/main.go

@@ -14,6 +14,7 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
+	"errors"
 	"flag"
 	"fmt"
 	"math/big"
@@ -191,7 +192,7 @@ func pemBlockForKey(priv interface{}) (*pem.Block, error) {
 		}
 		return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}, nil
 	default:
-		return nil, fmt.Errorf("unknown key type")
+		return nil, errors.New("unknown key type")
 	}
 }
 

cmd/syncthing/auditservice.go

@@ -1,69 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"encoding/json"
-	"io"
-
-	"github.com/syncthing/syncthing/lib/events"
-)
-
-// The auditService subscribes to events and writes these in JSON format, one
-// event per line, to the specified writer.
-type auditService struct {
-	w       io.Writer     // audit destination
-	stop    chan struct{} // signals time to stop
-	started chan struct{} // signals startup complete
-	stopped chan struct{} // signals stop complete
-}
-
-func newAuditService(w io.Writer) *auditService {
-	return &auditService{
-		w:       w,
-		stop:    make(chan struct{}),
-		started: make(chan struct{}),
-		stopped: make(chan struct{}),
-	}
-}
-
-// Serve runs the audit service.
-func (s *auditService) Serve() {
-	defer close(s.stopped)
-	sub := events.Default.Subscribe(events.AllEvents)
-	defer events.Default.Unsubscribe(sub)
-	enc := json.NewEncoder(s.w)
-
-	// We're ready to start processing events.
-	close(s.started)
-
-	for {
-		select {
-		case ev := <-sub.C():
-			enc.Encode(ev)
-		case <-s.stop:
-			return
-		}
-	}
-}
-
-// Stop stops the audit service.
-func (s *auditService) Stop() {
-	close(s.stop)
-}
-
-// WaitForStart returns once the audit service is ready to receive events, or
-// immediately if it's already running.
-func (s *auditService) WaitForStart() {
-	<-s.started
-}
-
-// WaitForStop returns once the audit service has stopped.
-// (Needed by the tests.)
-func (s *auditService) WaitForStop() {
-	<-s.stopped
-}

cmd/syncthing/blockprof.go

@@ -22,11 +22,15 @@ func init() {
 			panic("Couldn't find block profiler")
 		}
 		l.Debugln("Starting block profiling")
-		go saveBlockingProfiles(profiler)
+		go func() {
+			err := saveBlockingProfiles(profiler) // Only returns on error
+			l.Warnln("Block profiler failed:", err)
+			panic("Block profiler failed")
+		}()
 	}
 }
 
-func saveBlockingProfiles(profiler *pprof.Profile) {
+func saveBlockingProfiles(profiler *pprof.Profile) error {
 	runtime.SetBlockProfileRate(1)
 
 	t0 := time.Now()
@@ -35,16 +39,16 @@ func saveBlockingProfiles(profiler *pprof.Profile) {
 
 		fd, err := os.Create(fmt.Sprintf("block-%05d-%07d.pprof", syscall.Getpid(), startms))
 		if err != nil {
-			panic(err)
+			return err
 		}
 		err = profiler.WriteTo(fd, 0)
 		if err != nil {
-			panic(err)
+			return err
 		}
 		err = fd.Close()
 		if err != nil {
-			panic(err)
+			return err
 		}
-
 	}
+	return nil
 }

cmd/syncthing/cpuusage.go

@@ -1,59 +0,0 @@
-// Copyright (C) 2017 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"math"
-	"time"
-
-	metrics "github.com/rcrowley/go-metrics"
-)
-
-const cpuTickRate = 5 * time.Second
-
-type cpuService struct {
-	avg  metrics.EWMA
-	stop chan struct{}
-}
-
-func newCPUService() *cpuService {
-	return &cpuService{
-		// 10 second average. Magic alpha value comes from looking at EWMA package
-		// definitions of EWMA1, EWMA5. The tick rate *must* be five seconds (hard
-		// coded in the EWMA package).
-		avg:  metrics.NewEWMA(1 - math.Exp(-float64(cpuTickRate)/float64(time.Second)/10.0)),
-		stop: make(chan struct{}),
-	}
-}
-
-func (s *cpuService) Serve() {
-	// Initialize prevUsage to an actual value returned by cpuUsage
-	// instead of zero, because at least Windows returns a huge negative
-	// number here that then slowly increments...
-	prevUsage := cpuUsage()
-	ticker := time.NewTicker(cpuTickRate)
-	defer ticker.Stop()
-	for {
-		select {
-		case <-ticker.C:
-			curUsage := cpuUsage()
-			s.avg.Update(int64((curUsage - prevUsage) / time.Millisecond))
-			prevUsage = curUsage
-			s.avg.Tick()
-		case <-s.stop:
-			return
-		}
-	}
-}
-
-func (s *cpuService) Stop() {
-	close(s.stop)
-}
-
-func (s *cpuService) Rate() float64 {
-	return s.avg.Rate()
-}