Revert "lib/model: Add folders on start in model (#6135)"

This reverts commit bee7cce081.

.codecov.yml

@@ -0,0 +1,5 @@
+coverage:
+  range: "40...100"
+
+ignore:
+  - "**.pb.go"

.github/SECURITY.md

@@ -0,0 +1,48 @@
+## Reporting a Vulnerability
+
+If you believe that you've found a Syncthing-related security vulnerability, please report it by sending email to the address security@syncthing.net. The PGP key for security@syncthing.net (B683AD7B76CAB013) below can be used to send encrypted mail or to verify responses received from that address.
+
+```
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQENBFShFlIBCACsW346HYskhKhxrdZMjyU5Ntsjvg6/ogqINDPoL10/oaIP0G+t
+7zzC0K5Cq29ix43kNNLKTJNyPkdTeaJEcqslMUt6tovjHwoKJ073GP0W3KsNvBRg
+ffCZOAexGfOsBSL9KHaYGK67Py3TFgtN1H/EmboU1arrLfAMrmqOip++EGqOxjse
+gH0qk7Mk1TqEC5Xh3NGE7r1UobAlqdUv5E3v7U11NhAdP1zu/XZ/zvP5mwVQJMLv
+iZyeWGliNI8nEeRjYw+S85f4gq7H2mgoeNBN4WwwK1hhz9qpvCsgPW3XqlExTPI4
+1vM4PxpiFIuF0zuy2OwsmjrpTCZeBscr4Tj5ABEBAAG0K1N5bmN0aGluZyBTZWN1
+cml0eSA8c2VjdXJpdHlAc3luY3RoaW5nLm5ldD6JATgEEwECACIFAlShFlICGwMG
+CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELaDrXt2yrATB1UH/jnnIa6DekCA
+2V36N/2+pFSNLVWeoZQxZ9ne9S7WSubaoK4oCWiuChPSAy5hagnKnNA3a9wrz5iN
+0hgCA88NnTU18biZW6HX8xWEd3dnY3fX1sG0XdHuKlFria8ByfcrbShf/CttXkEd
+Y5qPHH9aLIMtksBS1MIsRYrOCHiLNYFCKlbjDDCGT3tuk/yaU7aBAFVPIag54afC
+zZAIvTIgRruLWkNT/sSEJx9ekhJzO/+pXNbSSHwhoj5OJh7sQjZWwPzNazelk48R
++ku8VpB5Wxgk2MnJPf1RxF+7saBAaeAVZsJcPN+Jp7u9LCFelIRn1ISsHbhLhyqL
+cPXqllltLCKJAhwEEAECAAYFAlShFlsACgkQSfWuwLzlJMcEpA//VvZYHGZgZMM0
+bBkYnjManYnJkHSQ2pHsjquSFH+fbfq2FxxTk/nQ/IAvBzt8NDTT3ylPOmsl4A8q
+r8BxsRNENhU7zDQrKC5NtYUrzXhPGo8qfDwkeLyqd2msUvHn7EH3PNgiN2QaFWxE
+21FqoXIpERKJRgRtv1SYZoMyNGjmT2hta1ZbwEfrPJnzjYhneoUGsRApG7p+uPqe
+4LRpbG3Fa2vBm/UWUrOe+6jPzvMokjIJbdK0IjXamFAzwYW5fSZaFa94mR6L5f5m
+X8Dhp66mBRTx7qk6ldEqptvaYGqihaWP1xbDaApQsGHQujtcBYGp+edlkabuW5h7
+stl/7QTFkEPqHT4ybxEX0rLoFUGq56WUlKp27z40keStaXMgfrsxJtkz66Xs8vU4
+7qZcLAcPsin+y0toqavtwtM/L7uCMu1yhbFRGJ+JL6saJAqzwS8l7r6E2R7OnVj1
+BdASgxx1TgzW6ZFW5p1Iy6mtpkBypsnp8s3UcP3GFRnQe9gi1EXHjzuZAUGafe4Q
+juvJ8t8xIcQMFuAylNIVyXvIWJoehqsVY9EBgVtE4y1SRh8XTT3Tddn8ab5fl7uh
+HWAY8cRlv6WIOhK8w8oroiYx0SID8jMeEwJBS4DL7qWtJMkDo8ZEJiB+Pkd963MX
+05QXt33AvJJ9PmbGCHkcH7198tCmA+e5AQ0EVKEWUgEIAPGczHpa6NdxY0pm+tIp
+btiA6gdPE70pJYgJTKX7siCQ2w770H3CBSKmqEXadr7WnfIgUYIDaSxadeGzB/Mn
+3SHCYRCqbA7mwu2k4wNNvCEM0xZqFAvaDJ4avlZ5oiMT8IFHKsjC77nkhmfXaIGt
+hn10H2MFADjvJYul7vR+Ghg1wGhTGWo2u7VVj9BI+AfvnWaouFI0cx2KNWEI/Ocj
+z6jk8nmC3yOEFQECM/hF4lkAOv9CQUa8UcvAr31trzawmV1iSsKjmVZgqd0N4T8f
+hUikqUPZGNCRcqEUffTzggIyGPbedFnZw9Di7o1xByxyTrZycemAVqaVGF+9nFLG
+pccAEQEAAYkBHwQYAQIACQUCVKEWUgIbDAAKCRC2g617dsqwEzrjB/9q0T8A4XUE
+p0g6xq86jhmh4jlEedxrfXUL3R6ejFtuKMThulxEP0xiQ7xLzBhOnvyxLCsVbjp8
+0JtJTVCq44UzUiIBuVNRoYG29uXuTUL2UtI27VhjFxMzLDwZL97tbGR6lzdM/+U5
+9PC+PIvS0yz13z2t3/x3KUOnBgxZnpy9h4AdKwjrNVtnQdGDXSKlJBLb57TFcS94
+f3roZ5Gpw3AWYSmSSiZWbhks2UNzYSQ84LAKlV1NkktO+qRc/pUqr6IxMjOKc7XP
+e8u1Nst6fN3GNqZOV+jUYs/fqrJgp1TUWjNTuf22Rl0Idz7XLPJKYFh9W7T/4MbU
+M7Q8GZuww1rk
+=No/v
+-----END PGP PUBLIC KEY BLOCK-----
+```

.gitignore

@@ -23,3 +23,4 @@ parts/
 stage/
 *.snap
 *.bz2
+/repos

.golangci.yml

@@ -0,0 +1,24 @@
+linters-settings:
+  maligned:
+    suggest-new: true
+
+linters:
+  enable-all: true
+  disable:
+    - goimports
+    - depguard
+    - lll
+    - gochecknoinits
+    - gochecknoglobals
+    - gofmt
+    - scopelint
+    - gocyclo
+    - funlen
+    - wsl
+
+service:
+  golangci-lint-version: 1.21.x
+  prepare:
+    - rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
+    - GO111MODULE=on go mod vendor
+    - go run build.go assets

AUTHORS

@@ -24,12 +24,14 @@ andresvia <andres.via@gmail.com>
 Andrew Dunham (andrew-d) <andrew@du.nham.ca>
 Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
 Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
+André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
 andyleap <andyleap@gmail.com>
 Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
 Antony Male (canton7) <antony.male@gmail.com>
 Aranjedeath <Aranjedeath@users.noreply.github.com>
 Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
-Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com>
+Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
+BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
 Bart De Vries (mogwa1) <devriesb@gmail.com>
 Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
 Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
@@ -37,12 +39,14 @@ Ben Shepherd (benshep) <bjashepherd@gmail.com>
 Ben Sidhom (bsidhom) <bsidhom@gmail.com>
 Benedikt Heine (bebehei) <bebe@bebehei.de>
 Benedikt Morbach <benedikt.morbach@googlemail.com>
+Benno Fünfstück <benno.fuenfstueck@gmail.com>
 Benny Ng (tpng) <benny.tpng@gmail.com>
+Boris Rybalkin <ribalkin@gmail.com>
 Brandon Philips (philips) <brandon@ifup.org>
 Brendan Long (brendanlong) <self@brendanlong.com>
 Brian R. Becker (brbecker) <brbecker@gmail.com>
 Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
-Carsten Hagemann (Moter8) <moter8@gmail.com>
+Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
 Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
 Cedric Staniewski (xduugu) <cedric@gmx.ca>
 Chris Howie (cdhowie) <me@chrishowie.com>
@@ -50,6 +54,7 @@ Chris Joel (cdata) <chris@scriptolo.gy>
 Chris Tonkinson <chris@masterbran.ch>
 chucic <chucic@seznam.cz>
 Colin Kennedy (moshen) <moshen.colin@gmail.com>
+Cromefire_ <tim.l@nghorst.net>
 Dale Visser <dale.visser@live.com>
 Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
 Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
@@ -58,25 +63,32 @@ Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
 David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
 Denis A. (dva) <denisva@gmail.com>
 Dennis Wilson (snnd) <dw@risu.io>
+dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com>
+dependabot[bot] <dependabot[bot]@users.noreply.github.com>
 derekriemer <derek.riemer@colorado.edu>
+desbma <desbma@users.noreply.github.com>
 Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
 Dominik Heidler (asdil12) <dominik@heidler.eu>
 Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
 Elliot Huffman <thelich2@gmail.com>
 Emil Hessman (ceh) <emil@hessman.se>
 Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
+Evgeny Kuznetsov <evgeny@kuznetsov.md>
 Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
 Felix Ableitner (Nutomic) <me@nutomic.com>
 Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
 Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
 Frank Isemann (fti7) <frank@isemann.name>
+georgespatton <georgespatton@users.noreply.github.com>
 Gilli Sigurdsson (gillisig) <gilli@vx.is>
 Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
 Han Boetes <han@boetes.org>
 Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
 Heiko Zuerker (Smiley73) <heiko@zuerker.org>
+Hugo Locurcio <hugo.locurcio@hugo.pro>
 Iain Barnett <iainspeed@gmail.com>
 Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
+Iskander Sharipov (Alex) <quasilyte@gmail.com>
 Jaakko Hannikainen (jgke) <jgke@jgke.fi>
 Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
 Jake Peterson (acogdev) <jake@acogdev.com>
@@ -92,8 +104,11 @@ Jochen Voss (seehuhn) <voss@seehuhn.de>
 Johan Andersson <j@i19.se>
 Johan Vromans (sciurius) <jvromans@squirrel.nl>
 John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
+Jonas Thelemann <e-mail@jonas-thelemann.de>
 Jonathan Cross <jcross@gmail.com>
 Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
+Jörg Thalheim <Mic92@users.noreply.github.com>
+Kalle Laine <pahakalle@protonmail.com>
 Karol Różycki (krozycki) <rozycki.karol@gmail.com>
 Keith Turner <kturner@apache.org>
 Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
@@ -118,7 +133,9 @@ Mark Pulford (mpx) <mark@kyne.com.au>
 Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
 Matic Potočnik <hairyfotr@gmail.com>
 Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
+Matt Robenolt <matt@ydekproductions.com>
 Matteo Ruina <matteo.ruina@gmail.com>
+Maurizio Tomasi <ziotom78@gmail.com>
 Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
 MaximAL <almaximal@ya.ru>
 Maxime Thirouin <m@moox.io>
@@ -127,16 +144,22 @@ Michael Ploujnikov (plouj) <ploujj@gmail.com>
 Michael Tilli (pyfisch) <pyfisch@gmail.com>
 Mike Boone <mike@boonedocks.net>
 MikeLund <MikeLund@users.noreply.github.com>
+Mingxuan Lin <gdlmx@users.noreply.github.com>
 Nate Morrison (nrm21) <natemorrison@gmail.com>
 Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
+Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
 Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
 Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
 Nils Jakobi (thunderstorm99) <jakobi.nils@gmail.com>
+Nitroretro <43112364+Nitroretro@users.noreply.github.com>
 NoLooseEnds <jon.koslung@gmail.com>
+otbutz <tbutz@optitool.de>
 Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
 Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
 Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
+Paweł Rozlach <vespian@users.noreply.github.com>
 perewa <cavalcante.ten@gmail.com>
+Peter Badida <KeyWeeUsr@users.noreply.github.com>
 Peter Dave Hello <hsu@peterdavehello.org>
 Peter Hoeg (peterhoeg) <peter@speartail.com>
 Peter Marquardt (wwwutz) <wwwutz@gmail.com> <wwwutz@googlemail.com>
@@ -158,7 +181,7 @@ Sergey Mishin (ralder) <ralder@yandex.ru>
 Simon Frei (imsodin) <freisim93@gmail.com>
 Sly_tom_cat <slytomcat@mail.ru>
 Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
-Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
+Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
 Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
 Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
 Thomas Hipp <thomashipp@gmail.com>
@@ -166,6 +189,7 @@ Tim Abell (timabell) <tim@timwise.co.uk>
 Tim Howes (timhowes) <timhowes@berkeley.edu>
 Tobias Nygren (tnn2) <tnn@nygren.pp.se>
 Tobias Tom (tobiastom) <t.tom@succont.de>
+Tom Jakubowski <tom@crystae.net>
 Tomas Cerveny (kozec) <kozec@kozec.com>
 Tommy Thorn <tommy-github-email@thorn.ws>
 Tully Robinson (tojrobinson) <tully@tojr.org>

Dockerfile

@@ -1,12 +1,12 @@
-FROM golang:1.10 AS builder
+FROM golang:1.13 AS builder
 
-WORKDIR /go/src/github.com/syncthing/syncthing
+WORKDIR /src
 COPY . .
 
 ENV CGO_ENABLED=0
 ENV BUILD_HOST=syncthing.net
 ENV BUILD_USER=docker
-RUN rm -f syncthing && go run build.go build syncthing
+RUN rm -f syncthing && go run build.go -no-upgrade build syncthing
 
 FROM alpine
 
@@ -14,37 +14,15 @@ EXPOSE 8384 22000 21027/udp
 
 VOLUME ["/var/syncthing"]
 
-RUN apk add --no-cache ca-certificates
+RUN apk add --no-cache ca-certificates su-exec
 
-COPY --from=builder /go/src/github.com/syncthing/syncthing/syncthing /bin/syncthing
+COPY --from=builder /src/syncthing /bin/syncthing
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
 
-RUN apk add --no-cache su-exec
-
-ENV STNOUPGRADE=1
-ENV PUSR=syncthing
-ENV PUID=1000
-ENV PGRP=syncthing
-ENV PGID=1000
+ENV PUID=1000 PGID=1000 HOME=/var/syncthing
 
 HEALTHCHECK --interval=1m --timeout=10s \
   CMD nc -z localhost 8384 || exit 1
 
-ENTRYPOINT true \
- && ( getent group "${PGRP}" >/dev/null \
-      || addgroup \
-          -g "${PGID}" \
-          "${PGRP}" \
-    ) \
- && ( getent passwd "${PUSR}" >/dev/null \
-      || adduser \
-          -h /var/syncthing \
-          -G "${PGRP}" \
-          -u "${PUID}" \
-          "${PUSR}" \
-    ) \
- && chown "${PUSR}:${PGRP}" /var/syncthing \
- && su-exec "${PUSR}:${PGRP}" \
-     /bin/syncthing \
-       -home /var/syncthing/config \
-       -gui-address 0.0.0.0:8384 \
- && true
+ENV STGUIADDRESS=0.0.0.0:8384
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]

Dockerfile.stdiscosrv

@@ -0,0 +1,28 @@
+FROM golang:1.13 AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f stdiscosrv && go run build.go -no-upgrade build stdiscosrv
+
+FROM alpine
+
+EXPOSE 19200 8443
+
+VOLUME ["/var/stdiscosrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/stdiscosrv /bin/stdiscosrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/stdiscosrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 8443 || exit 1
+
+WORKDIR /var/stdiscosrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/stdiscosrv"]

Dockerfile.strelaysrv

@@ -0,0 +1,28 @@
+FROM golang:1.13 AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f strelaysrv && go run build.go -no-upgrade build strelaysrv
+
+FROM alpine
+
+EXPOSE 22067 22070
+
+VOLUME ["/var/strelaysrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/strelaysrv /bin/strelaysrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/strelaysrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 22067 || exit 1
+
+WORKDIR /var/strelaysrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaysrv"]

README-Docker.md

@@ -1,39 +1,34 @@
 # Docker Container for Syncthing
 
 Use the Dockerfile in this repo, or pull the `syncthing/syncthing` image
-from Docker Hub. Use volumes to have the synchronized files available on the
-host.
+from Docker Hub.
 
-The exposed volumes are by default:
-
-    /var/syncthing/config   - the configuration and index directory into the Container
-    /var/syncthing          - the default sync folder into the Container
-
-You can add more folders and map them as you prefer.
+Use the `/var/syncthing` volume to have the synchronized files available on the
+host. You can add more folders and map them as you prefer.
 
 Note that Syncthing runs as UID 1000 and GID 1000 by default. These may be
 altered with the ``PUID`` and ``PGID`` environment variables.
 
-Example usage:
+## Example Usage
 
 ```
 $ docker pull syncthing/syncthing
 $ docker run -p 8384:8384 -p 22000:22000 \
-    -v /wherever/st-cfg:/var/syncthing/config \
     -v /wherever/st-sync:/var/syncthing \
     syncthing/syncthing:latest
 ```
 
-Note that local device discovery will not work with the above command resulting
-in poor local transfer rates if local device addresses are not manually
-configured.
+## Discovery
+
+Note that local device discovery will not work with the above command,
+resulting in poor local transfer rates if local device addresses are not
+manually configured.
 
 To allow local discovery, the docker host network can be used instead:
 
 ```
 $ docker pull syncthing/syncthing
 $ docker run --network=host \
-    -v /wherever/st-cfg:/var/syncthing/config \
     -v /wherever/st-sync:/var/syncthing \
     syncthing/syncthing:latest
 ```
@@ -41,3 +36,24 @@ $ docker run --network=host \
 Be aware that syncthing alone is now in control of what interfaces and ports it
 listens on. You can edit the syncthing configuration to change the defaults if
 there are conflicts.
+
+## GUI Security
+
+By default Syncthing inside the Docker image listens on 0.0.0.0:8384 to
+allow GUI connections via the Docker proxy. This is set by the
+`STGUIADDRESS` environment variable in the Dockerfile, as it differs from
+what Syncthing would otherwise use by default. This means you should set up
+authentication in the GUI, like for any other externally reachable Syncthing
+instance. If you do not require the GUI, or you use host networking, you can
+unset the `STGUIADDRESS` variable to have Syncthing fall back to listening
+on 127.0.0.1:
+
+```
+$ docker pull syncthing/syncthing
+$ docker run -e STGUIADDRESS= \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+With the environment variable unset Syncthing will follow what is set in the
+configuration file / GUI settings dialog.

README.md

@@ -2,7 +2,6 @@
 
 ---
 
-[![Latest Downloads](https://img.shields.io/badge/latest-downloads-brightgreen.svg?style=flat-square)](https://build.syncthing.net/latest/)
 [![Latest Linux & Cross Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildLinuxCross.svg?style=flat-square&label=linux+%26+cross+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildLinuxCross&guest=1)
 [![Latest Windows Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildWindows.svg?style=flat-square&label=windows+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildWindows&guest=1)
 [![Latest Mac Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildMac.svg?style=flat-square&label=mac+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildMac&guest=1)
@@ -63,6 +62,10 @@ There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory][3]. There are also several [GUI
 implementations][11] for Windows, Mac and Linux.
 
+## Docker
+
+To run Syncthing in Docker, see [the Docker README][16].
+
 ## Vote on features/bugs
 
 We'd like to encourage you to [vote][12] on issues that matter to you.
@@ -111,4 +114,5 @@ All code is licensed under the [MPLv2 License][7].
 [13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
 [14]: assets/logo-text-128.png
 [15]: https://syncthing.net/
+[16]: https://github.com/syncthing/syncthing/blob/master/README-Docker.md
 

build.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"os/exec"
 	"os/user"
+	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -39,14 +40,16 @@ var (
 	goos          string
 	noupgrade     bool
 	version       string
+	goCmd         string
 	goVersion     float64
 	race          bool
 	debug         = os.Getenv("BUILDDEBUG") != ""
-	noBuildGopath bool
 	extraTags     string
 	installSuffix string
 	pkgdir        string
+	cc            string
 	debugBinary   bool
+	coverage      bool
 	timeout       = "120s"
 )
 
@@ -54,11 +57,13 @@ type target struct {
 	name              string
 	debname           string
 	debdeps           []string
+	debpre            string
 	debpost           string
 	description       string
-	buildPkg          string
+	buildPkgs         []string
 	binaryName        string
 	archiveFiles      []archiveFile
+	systemdServices   []string
 	installationFiles []archiveFile
 	tags              []string
 }
@@ -72,9 +77,8 @@ type archiveFile struct {
 var targets = map[string]target{
 	"all": {
 		// Only valid for the "build" and "install" commands as it lacks all
-		// the archive creation stuff.
-		buildPkg: "github.com/syncthing/syncthing/cmd/...",
-		tags:     []string{"purego"},
+		// the archive creation stuff. buildPkgs gets filled out in init()
+		tags: []string{"purego"},
 	},
 	"syncthing": {
 		// The default target for "build", "install", "tar", "zip", "deb", etc.
@@ -83,7 +87,7 @@ var targets = map[string]target{
 		debdeps:     []string{"libc6", "procps"},
 		debpost:     "script/post-upgrade",
 		description: "Open Source Continuous File Synchronization",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/syncthing",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/syncthing"},
 		binaryName:  "syncthing", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -111,14 +115,23 @@ var targets = map[string]target{
 			{src: "etc/linux-systemd/system/syncthing-resume.service", dst: "deb/lib/systemd/system/syncthing-resume.service", perm: 0644},
 			{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0644},
 			{src: "etc/firewall-ufw/syncthing", dst: "deb/etc/ufw/applications.d/syncthing", perm: 0644},
+			{src: "etc/linux-desktop/syncthing-start.desktop", dst: "deb/usr/share/applications/syncthing-start.desktop", perm: 0644},
+			{src: "etc/linux-desktop/syncthing-ui.desktop", dst: "deb/usr/share/applications/syncthing-ui.desktop", perm: 0644},
+			{src: "assets/logo-32.png", dst: "deb/usr/share/icons/hicolor/32x32/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-64.png", dst: "deb/usr/share/icons/hicolor/64x64/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-128.png", dst: "deb/usr/share/icons/hicolor/128x128/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-256.png", dst: "deb/usr/share/icons/hicolor/256x256/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-512.png", dst: "deb/usr/share/icons/hicolor/512x512/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-only.svg", dst: "deb/usr/share/icons/hicolor/scalable/apps/syncthing.svg", perm: 0644},
 		},
 	},
 	"stdiscosrv": {
 		name:        "stdiscosrv",
 		debname:     "syncthing-discosrv",
 		debdeps:     []string{"libc6"},
+		debpre:      "cmd/stdiscosrv/scripts/preinst",
 		description: "Syncthing Discovery Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/stdiscosrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/stdiscosrv"},
 		binaryName:  "stdiscosrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -126,12 +139,17 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
+		systemdServices: []string{
+			"cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service",
+		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
 			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0644},
-			{src: "cmd/stdiscosrv/LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
+			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0644},
 			{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-discosrv", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv", dst: "deb/etc/ufw/applications.d/stdiscosrv", perm: 0644},
 		},
 		tags: []string{"purego"},
 	},
@@ -139,21 +157,29 @@ var targets = map[string]target{
 		name:        "strelaysrv",
 		debname:     "syncthing-relaysrv",
 		debdeps:     []string{"libc6"},
+		debpre:      "cmd/strelaysrv/scripts/preinst",
 		description: "Syncthing Relay Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/strelaysrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaysrv"},
 		binaryName:  "strelaysrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
 			{src: "cmd/strelaysrv/README.md", dst: "README.txt", perm: 0644},
 			{src: "cmd/strelaysrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
+		systemdServices: []string{
+			"cmd/strelaysrv/etc/linux-systemd/strelaysrv.service",
+		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
 			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0644},
 			{src: "cmd/strelaysrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
+			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0644},
 			{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0644},
+			{src: "cmd/strelaysrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-relaysrv", perm: 0644},
+			{src: "cmd/strelaysrv/etc/firewall-ufw/strelaysrv", dst: "deb/etc/ufw/applications.d/strelaysrv", perm: 0644},
 		},
 	},
 	"strelaypoolsrv": {
@@ -161,7 +187,7 @@ var targets = map[string]target{
 		debname:     "syncthing-relaypoolsrv",
 		debdeps:     []string{"libc6"},
 		description: "Syncthing Relay Pool Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/strelaypoolsrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaypoolsrv"},
 		binaryName:  "strelaypoolsrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -178,7 +204,31 @@ var targets = map[string]target{
 	},
 }
 
+// These are repos we need to clone to run "go generate"
+
+type dependencyRepo struct {
+	path   string
+	repo   string
+	commit string
+}
+
+var dependencyRepos = []dependencyRepo{
+	{path: "xdr", repo: "https://github.com/calmh/xdr.git", commit: "08e072f9cb16"},
+}
+
 func init() {
+	all := targets["all"]
+	pkgs, _ := filepath.Glob("cmd/*")
+	for _, pkg := range pkgs {
+		pkg = filepath.Base(pkg)
+		if strings.HasPrefix(pkg, ".") {
+			// ignore dotfiles
+			continue
+		}
+		all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/cmd/%s", pkg))
+	}
+	targets["all"] = all
+
 	// The "syncthing" target includes a few more files found in the "etc"
 	// and "extra" dirs.
 	syncthingPkg := targets["syncthing"]
@@ -206,39 +256,6 @@ func main() {
 		}()
 	}
 
-	if gopath := gopath(); gopath == "" {
-		gopath, err := temporaryBuildDir()
-		if err != nil {
-			log.Fatal(err)
-		}
-		os.Setenv("GOPATH", gopath)
-		log.Println("GOPATH is", gopath)
-		if !noBuildGopath {
-			if err := buildGOPATH(gopath); err != nil {
-				log.Fatal(err)
-			}
-			lazyRebuildAssets()
-		}
-	} else {
-		inside := false
-		wd, _ := os.Getwd()
-		wd, _ = filepath.EvalSymlinks(wd)
-		for _, p := range filepath.SplitList(gopath) {
-			p, _ = filepath.EvalSymlinks(p)
-			if filepath.Join(p, "src/github.com/syncthing/syncthing") == wd {
-				inside = true
-				break
-			}
-		}
-		if !inside {
-			fmt.Println("You seem to have GOPATH set but the Syncthing source not placed correctly within it, which may cause problems.")
-		}
-	}
-
-	// Set path to $GOPATH/bin:$PATH so that we can for sure find tools we
-	// might have installed during "build.go setup".
-	os.Setenv("PATH", fmt.Sprintf("%s%cbin%c%s", os.Getenv("GOPATH"), os.PathSeparator, os.PathListSeparator, os.Getenv("PATH")))
-
 	// Invoking build.go with no parameters at all builds everything (incrementally),
 	// which is what you want for maximum error checking during development.
 	if flag.NArg() == 0 {
@@ -262,9 +279,6 @@ func main() {
 
 func runCommand(cmd string, target target) {
 	switch cmd {
-	case "setup":
-		setup()
-
 	case "install":
 		var tags []string
 		if noupgrade {
@@ -312,9 +326,6 @@ func runCommand(cmd string, target target) {
 	case "snap":
 		buildSnap(target)
 
-	case "clean":
-		clean()
-
 	case "vet":
 		metalintShort()
 
@@ -327,13 +338,6 @@ func runCommand(cmd string, target target) {
 	case "version":
 		fmt.Println(getVersion())
 
-	case "gopath":
-		gopath, err := temporaryBuildDir()
-		if err != nil {
-			log.Fatal(err)
-		}
-		fmt.Println(gopath)
-
 	default:
 		log.Fatalf("Unknown command %q", cmd)
 	}
@@ -342,64 +346,41 @@ func runCommand(cmd string, target target) {
 func parseFlags() {
 	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
 	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
+	flag.StringVar(&goCmd, "gocmd", "go", "Specify `go` command")
 	flag.BoolVar(&noupgrade, "no-upgrade", noupgrade, "Disable upgrade functionality")
 	flag.StringVar(&version, "version", getVersion(), "Set compiled in version string")
 	flag.BoolVar(&race, "race", race, "Use race detector")
-	flag.BoolVar(&noBuildGopath, "no-build-gopath", noBuildGopath, "Don't build GOPATH, assume it's OK")
 	flag.StringVar(&extraTags, "tags", extraTags, "Extra tags, space separated")
 	flag.StringVar(&installSuffix, "installsuffix", installSuffix, "Install suffix, optional")
 	flag.StringVar(&pkgdir, "pkgdir", "", "Set -pkgdir parameter for `go build`")
+	flag.StringVar(&cc, "cc", os.Getenv("CC"), "Set CC environment variable for `go build`")
 	flag.BoolVar(&debugBinary, "debug-binary", debugBinary, "Create unoptimized binary to use with delve, set -gcflags='-N -l' and omit -ldflags")
+	flag.BoolVar(&coverage, "coverage", coverage, "Write coverage profile of tests to coverage.txt")
 	flag.Parse()
 }
 
-func setup() {
-	packages := []string{
-		"github.com/alecthomas/gometalinter",
-		"github.com/AlekSi/gocov-xml",
-		"github.com/axw/gocov/gocov",
-		"github.com/FiloSottile/gvt",
-		"github.com/golang/lint/golint",
-		"github.com/gordonklaus/ineffassign",
-		"github.com/mdempsky/unconvert",
-		"github.com/mitchellh/go-wordwrap",
-		"github.com/opennota/check/cmd/...",
-		"github.com/tsenart/deadcode",
-		"golang.org/x/net/html",
-		"golang.org/x/tools/cmd/cover",
-		"honnef.co/go/tools/cmd/gosimple",
-		"honnef.co/go/tools/cmd/staticcheck",
-		"honnef.co/go/tools/cmd/unused",
-		"github.com/josephspurrier/goversioninfo",
-	}
-	for _, pkg := range packages {
-		fmt.Println(pkg)
-		runPrint("go", "get", "-u", pkg)
-	}
-
-	runPrint("go", "install", "-v", "github.com/syncthing/syncthing/vendor/github.com/gogo/protobuf/protoc-gen-gogofast")
-}
-
 func test(pkgs ...string) {
 	lazyRebuildAssets()
 
-	useRace := runtime.GOARCH == "amd64"
-	switch runtime.GOOS {
-	case "darwin", "linux", "freebsd", "windows":
-	default:
-		useRace = false
+	args := []string{"test", "-short", "-timeout", timeout, "-tags", "purego"}
+
+	if runtime.GOARCH == "amd64" {
+		switch runtime.GOOS {
+		case "darwin", "linux", "freebsd": // , "windows": # See https://github.com/golang/go/issues/27089
+			args = append(args, "-race")
+		}
 	}
 
-	if useRace {
-		runPrint("go", append([]string{"test", "-short", "-race", "-timeout", timeout, "-tags", "purego"}, pkgs...)...)
-	} else {
-		runPrint("go", append([]string{"test", "-short", "-timeout", timeout, "-tags", "purego"}, pkgs...)...)
+	if coverage {
+		args = append(args, "-covermode", "atomic", "-coverprofile", "coverage.txt", "-coverpkg", strings.Join(pkgs, ","))
 	}
+
+	runPrint(goCmd, append(args, pkgs...)...)
 }
 
 func bench(pkgs ...string) {
 	lazyRebuildAssets()
-	runPrint("go", append([]string{"test", "-run", "NONE", "-bench", "."}, pkgs...)...)
+	runPrint(goCmd, append([]string{"test", "-run", "NONE", "-bench", "."}, pkgs...)...)
 }
 
 func install(target target, tags []string) {
@@ -413,11 +394,9 @@ func install(target target, tags []string) {
 	}
 	os.Setenv("GOBIN", filepath.Join(cwd, "bin"))
 
-	args := []string{"install", "-v"}
-	args = appendParameters(args, tags, target)
-
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
+	os.Setenv("CC", cc)
 
 	// On Windows generate a special file which the Go compiler will
 	// automatically use when generating Windows binaries to set things like
@@ -430,21 +409,23 @@ func install(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	runPrint("go", args...)
+	for _, pkg := range target.buildPkgs {
+		args := []string{"install", "-v"}
+		args = appendParameters(args, tags, pkg)
+
+		runPrint(goCmd, args...)
+	}
 }
 
 func build(target target, tags []string) {
 	lazyRebuildAssets()
-
 	tags = append(target.tags, tags...)
 
 	rmr(target.BinaryName())
 
-	args := []string{"build", "-i", "-v"}
-	args = appendParameters(args, tags, target)
-
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
+	os.Setenv("CC", cc)
 
 	// On Windows generate a special file which the Go compiler will
 	// automatically use when generating Windows binaries to set things like
@@ -461,10 +442,15 @@ func build(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	runPrint("go", args...)
+	for _, pkg := range target.buildPkgs {
+		args := []string{"build", "-v"}
+		args = appendParameters(args, tags, pkg)
+
+		runPrint(goCmd, args...)
+	}
 }
 
-func appendParameters(args []string, tags []string, target target) []string {
+func appendParameters(args []string, tags []string, pkg string) []string {
 	if pkgdir != "" {
 		args = append(args, "-pkgdir", pkgdir)
 	}
@@ -480,7 +466,7 @@ func appendParameters(args []string, tags []string, target target) []string {
 
 	if !debugBinary {
 		// Regular binaries get version tagged and skip some debug symbols
-		args = append(args, "-ldflags", ldflags())
+		args = append(args, "-ldflags", ldflags(path.Base(pkg)))
 	} else {
 		// -gcflags to disable optimizations and inlining. Skip -ldflags
 		// because `Could not launch program: decoding dwarf section info at
@@ -489,7 +475,7 @@ func appendParameters(args []string, tags []string, target target) []string {
 		args = append(args, "-gcflags", "-N -l")
 	}
 
-	return append(args, target.buildPkg)
+	return append(args, pkg)
 }
 
 func buildTar(target target) {
@@ -596,9 +582,15 @@ func buildDeb(target target) {
 	for _, dep := range target.debdeps {
 		args = append(args, "-d", dep)
 	}
+	for _, service := range target.systemdServices {
+		args = append(args, "--deb-systemd", service)
+	}
 	if target.debpost != "" {
 		args = append(args, "--after-upgrade", target.debpost)
 	}
+	if target.debpre != "" {
+		args = append(args, "--before-install", target.debpre)
+	}
 	runPrint("fpm", args...)
 }
 
@@ -731,6 +723,7 @@ func listFiles(dir string) []string {
 		if err != nil {
 			return err
 		}
+
 		if fi.Mode().IsRegular() {
 			res = append(res, path)
 		}
@@ -741,7 +734,7 @@ func listFiles(dir string) []string {
 
 func rebuildAssets() {
 	os.Setenv("SOURCE_DATE_EPOCH", fmt.Sprint(buildStamp()))
-	runPrint("go", "generate", "github.com/syncthing/syncthing/lib/auto", "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto")
+	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/auto", "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto")
 }
 
 func lazyRebuildAssets() {
@@ -777,12 +770,28 @@ func shouldRebuildAssets(target, srcdir string) bool {
 }
 
 func proto() {
-	runPrint("go", "generate", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/stdiscosrv")
+	pv := protobufVersion()
+	dependencyRepos = append(dependencyRepos,
+		dependencyRepo{path: "protobuf", repo: "https://github.com/gogo/protobuf.git", commit: pv},
+	)
+
+	runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
+	os.MkdirAll("repos", 0755)
+	for _, dep := range dependencyRepos {
+		path := filepath.Join("repos", dep.path)
+		if _, err := os.Stat(path); err != nil {
+			runPrintInDir("repos", "git", "clone", dep.repo, dep.path)
+		} else {
+			runPrintInDir(path, "git", "fetch")
+		}
+		runPrintInDir(path, "git", "checkout", dep.commit)
+	}
+	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/stdiscosrv")
 }
 
 func translate() {
 	os.Chdir("gui/default/assets/lang")
-	runPipe("lang-en-new.json", "go", "run", "../../../../script/translate.go", "lang-en.json", "../../../")
+	runPipe("lang-en-new.json", goCmd, "run", "../../../../script/translate.go", "lang-en.json", "../../../")
 	os.Remove("lang-en.json")
 	err := os.Rename("lang-en-new.json", "lang-en.json")
 	if err != nil {
@@ -793,15 +802,10 @@ func translate() {
 
 func transifex() {
 	os.Chdir("gui/default/assets/lang")
-	runPrint("go", "run", "../../../../script/transifexdl.go")
+	runPrint(goCmd, "run", "../../../../script/transifexdl.go")
 }
 
-func clean() {
-	rmr("bin")
-	rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/syncthing", goos, goarch)))
-}
-
-func ldflags() string {
+func ldflags(program string) string {
 	sep := '='
 	if goVersion > 0 && goVersion < 1.5 {
 		sep = ' '
@@ -809,10 +813,14 @@ func ldflags() string {
 
 	b := new(bytes.Buffer)
 	b.WriteString("-w")
-	fmt.Fprintf(b, " -X main.Version%c%s", sep, version)
-	fmt.Fprintf(b, " -X main.BuildStamp%c%d", sep, buildStamp())
-	fmt.Fprintf(b, " -X main.BuildUser%c%s", sep, buildUser())
-	fmt.Fprintf(b, " -X main.BuildHost%c%s", sep, buildHost())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Version%c%s", sep, version)
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp%c%d", sep, buildStamp())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User%c%s", sep, buildUser())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Host%c%s", sep, buildHost())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Program%c%s", sep, program)
+	if v := os.Getenv("EXTRA_LDFLAGS"); v != "" {
+		fmt.Fprintf(b, " %s", v)
+	}
 	return b.String()
 }
 
@@ -995,6 +1003,10 @@ func runError(cmd string, args ...string) ([]byte, error) {
 }
 
 func runPrint(cmd string, args ...string) {
+	runPrintInDir(".", cmd, args...)
+}
+
+func runPrintInDir(dir string, cmd string, args ...string) {
 	if debug {
 		t0 := time.Now()
 		log.Println("runPrint:", cmd, strings.Join(args, " "))
@@ -1005,6 +1017,7 @@ func runPrint(cmd string, args ...string) {
 	ecmd := exec.Command(cmd, args...)
 	ecmd.Stdout = os.Stdout
 	ecmd.Stderr = os.Stderr
+	ecmd.Dir = dir
 	err := ecmd.Run()
 	if err != nil {
 		log.Fatal(err)
@@ -1224,12 +1237,12 @@ func windowsCodesign(file string) {
 
 func metalint() {
 	lazyRebuildAssets()
-	runPrint("go", "test", "-run", "Metalint", "./meta")
+	runPrint(goCmd, "test", "-run", "Metalint", "./meta")
 }
 
 func metalintShort() {
 	lazyRebuildAssets()
-	runPrint("go", "test", "-short", "-run", "Metalint", "./meta")
+	runPrint(goCmd, "test", "-short", "-run", "Metalint", "./meta")
 }
 
 func temporaryBuildDir() (string, error) {
@@ -1258,93 +1271,17 @@ func temporaryBuildDir() (string, error) {
 	return filepath.Join(tmpDir, base), nil
 }
 
-func buildGOPATH(gopath string) error {
-	pkg := filepath.Join(gopath, "src/github.com/syncthing/syncthing")
-	dirs := []string{"cmd", "gui", "lib", "meta", "script", "test", "vendor"}
-
-	if debug {
-		t0 := time.Now()
-		log.Println("build temporary GOPATH in", gopath)
-		defer func() {
-			log.Println("... in", time.Since(t0))
-		}()
-	}
-
-	// Walk the sources and copy the files into the temporary GOPATH.
-	// Remember which files are supposed to be present so we can clean
-	// out everything else in the next step. The copyFile() step will
-	// only actually copy the file if it doesn't exist or the contents
-	// differ.
-
-	exists := map[string]struct{}{}
-	for _, dir := range dirs {
-		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			if info.IsDir() {
-				return nil
-			}
-
-			dst := filepath.Join(pkg, path)
-			exists[dst] = struct{}{}
-
-			if err := copyFile(path, dst, info.Mode()); err != nil {
-				return err
-			}
-
-			return nil
-		})
-		if err != nil {
-			return err
-		}
-	}
-
-	// Walk the temporary GOPATH and remove any files that we wouldn't
-	// have copied there in the previous step.
-
-	filepath.Walk(pkg, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if info.IsDir() {
-			return nil
-		}
-		if _, ok := exists[path]; !ok {
-			os.Remove(path)
-		}
-		return nil
-	})
-
-	return nil
-}
-
-func gopath() string {
-	if gopath := os.Getenv("GOPATH"); gopath != "" {
-		// The env var is set, use that.
-		return gopath
-	}
-
-	// Ask Go what it thinks.
-	bs, err := runError("go", "env", "GOPATH")
-	if err != nil {
-		return ""
-	}
-
-	// We got something. Check if we are in fact available in that location.
-	gopath := string(bs)
-	if _, err := os.Stat(filepath.Join(gopath, "src/github.com/syncthing/syncthing/build.go")); err == nil {
-		// That seems to be the gopath.
-		return gopath
-	}
-
-	// The gopath is not valid.
-	return ""
-}
-
 func (t target) BinaryName() string {
 	if goos == "windows" {
 		return t.binaryName + ".exe"
 	}
 	return t.binaryName
 }
+
+func protobufVersion() string {
+	bs, err := runError(goCmd, "list", "-f", "{{.Version}}", "-m", "github.com/gogo/protobuf")
+	if err != nil {
+		log.Fatal("Getting protobuf version:", err)
+	}
+	return string(bs)
+}

cmd/stbench/main.go

@@ -1,143 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// This doesn't build on Windows due to the Rusage stuff.
-
-// +build !windows
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"runtime"
-	"syscall"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/rc"
-)
-
-var homeDir = "h1"
-var syncthingBin = "./bin/syncthing"
-var test = "scan"
-
-func main() {
-	flag.StringVar(&homeDir, "home", homeDir, "Home directory location")
-	flag.StringVar(&syncthingBin, "bin", syncthingBin, "Binary location")
-	flag.StringVar(&test, "test", test, "Test to run")
-	flag.Parse()
-
-	switch test {
-	case "scan":
-		// scan measures the resource usage required to perform the initial
-		// scan, without cleaning away the database first.
-		testScan()
-	}
-}
-
-// testScan starts a process and reports on the resource usage required to
-// perform the initial scan.
-func testScan() {
-	log.Println("Starting...")
-	p := rc.NewProcess("127.0.0.1:8081")
-	if err := p.Start(syncthingBin, "-home", homeDir, "-no-browser"); err != nil {
-		log.Println(err)
-		return
-	}
-	defer p.Stop()
-
-	wallTime := awaitScanComplete(p)
-
-	report(p, wallTime)
-}
-
-// awaitScanComplete waits for a folder to transition idle->scanning and
-// then scanning->idle and returns the time taken for the scan.
-func awaitScanComplete(p *rc.Process) time.Duration {
-	log.Println("Awaiting scan completion...")
-	var t0, t1 time.Time
-	lastEvent := 0
-loop:
-	for {
-		evs, err := p.Events(lastEvent)
-		if err != nil {
-			continue
-		}
-
-		for _, ev := range evs {
-			if ev.Type == "StateChanged" {
-				data := ev.Data.(map[string]interface{})
-				log.Println(ev)
-
-				if data["to"].(string) == "scanning" {
-					t0 = ev.Time
-					continue
-				}
-
-				if !t0.IsZero() && data["to"].(string) == "idle" {
-					t1 = ev.Time
-					break loop
-				}
-			}
-			lastEvent = ev.ID
-		}
-
-		time.Sleep(250 * time.Millisecond)
-	}
-
-	return t1.Sub(t0)
-}
-
-// report stops the given process and reports on its resource usage in two
-// ways: human readable to stderr, and CSV to stdout.
-func report(p *rc.Process, wallTime time.Duration) {
-	sv, err := p.SystemVersion()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	ss, err := p.SystemStatus()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	proc, err := p.Stop()
-	if err != nil {
-		return
-	}
-
-	rusage, ok := proc.SysUsage().(*syscall.Rusage)
-	if !ok {
-		return
-	}
-
-	log.Println("Version:", sv.Version)
-	log.Println("Alloc:", ss.Alloc/1024, "KiB")
-	log.Println("Sys:", ss.Sys/1024, "KiB")
-	log.Println("Goroutines:", ss.Goroutines)
-	log.Println("Wall time:", wallTime)
-	log.Println("Utime:", time.Duration(rusage.Utime.Nano()))
-	log.Println("Stime:", time.Duration(rusage.Stime.Nano()))
-	if runtime.GOOS == "darwin" {
-		// Darwin reports in bytes, Linux seems to report in KiB even
-		// though the manpage says otherwise.
-		rusage.Maxrss /= 1024
-	}
-	log.Println("MaxRSS:", rusage.Maxrss, "KiB")
-
-	fmt.Printf("%s,%d,%d,%d,%.02f,%.02f,%.02f,%d\n",
-		sv.Version,
-		ss.Alloc/1024,
-		ss.Sys/1024,
-		ss.Goroutines,
-		wallTime.Seconds(),
-		time.Duration(rusage.Utime.Nano()).Seconds(),
-		time.Duration(rusage.Stime.Nano()).Seconds(),
-		rusage.Maxrss)
-}

cmd/stcli/LICENSE

@@ -1,19 +0,0 @@
-Copyright (C) 2014 Audrius Butkevičius
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-- The above copyright notice and this permission notice shall be included in
-  all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

cmd/stcli/client.go

@@ -1,115 +1,95 @@
-// Copyright (C) 2014 Audrius Butkevičius
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
+	"fmt"
+	"net"
 	"net/http"
 	"strings"
 
-	"github.com/AudriusButkevicius/cli"
+	"github.com/syncthing/syncthing/lib/config"
 )
 
 type APIClient struct {
-	httpClient http.Client
-	endpoint   string
-	apikey     string
-	username   string
-	password   string
-	id         string
-	csrf       string
+	http.Client
+	cfg    config.GUIConfiguration
+	apikey string
 }
 
-var instance *APIClient
-
-func getClient(c *cli.Context) *APIClient {
-	if instance != nil {
-		return instance
-	}
-	endpoint := c.GlobalString("endpoint")
-	if !strings.HasPrefix(endpoint, "http") {
-		endpoint = "http://" + endpoint
-	}
+func getClient(cfg config.GUIConfiguration) *APIClient {
 	httpClient := http.Client{
 		Transport: &http.Transport{
 			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: c.GlobalBool("insecure"),
+				InsecureSkipVerify: true,
+			},
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial(cfg.Network(), cfg.Address())
 			},
 		},
 	}
-	client := APIClient{
-		httpClient: httpClient,
-		endpoint:   endpoint,
-		apikey:     c.GlobalString("apikey"),
-		username:   c.GlobalString("username"),
-		password:   c.GlobalString("password"),
+	return &APIClient{
+		Client: httpClient,
+		cfg:    cfg,
+		apikey: cfg.APIKey,
 	}
-
-	if client.apikey == "" {
-		request, err := http.NewRequest("GET", client.endpoint, nil)
-		die(err)
-		response := client.handleRequest(request)
-		client.id = response.Header.Get("X-Syncthing-ID")
-		if client.id == "" {
-			die("Failed to get device ID")
-		}
-		for _, item := range response.Cookies() {
-			if item.Name == "CSRF-Token-"+client.id[:5] {
-				client.csrf = item.Value
-				goto csrffound
-			}
-		}
-		die("Failed to get CSRF token")
-	csrffound:
-	}
-	instance = &client
-	return &client
 }
 
-func (client *APIClient) handleRequest(request *http.Request) *http.Response {
-	if client.apikey != "" {
-		request.Header.Set("X-API-Key", client.apikey)
+func (c *APIClient) Endpoint() string {
+	if c.cfg.Network() == "unix" {
+		return "http://unix/"
 	}
-	if client.username != "" || client.password != "" {
-		request.SetBasicAuth(client.username, client.password)
-	}
-	if client.csrf != "" {
-		request.Header.Set("X-CSRF-Token-"+client.id[:5], client.csrf)
+	url := c.cfg.URL()
+	if !strings.HasSuffix(url, "/") {
+		url += "/"
 	}
+	return url
+}
 
-	response, err := client.httpClient.Do(request)
-	die(err)
+func (c *APIClient) Do(req *http.Request) (*http.Response, error) {
+	req.Header.Set("X-API-Key", c.apikey)
+	resp, err := c.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	return resp, checkResponse(resp)
+}
 
+func (c *APIClient) Get(url string) (*http.Response, error) {
+	request, err := http.NewRequest("GET", c.Endpoint()+"rest/"+url, nil)
+	if err != nil {
+		return nil, err
+	}
+	return c.Do(request)
+}
+
+func (c *APIClient) Post(url, body string) (*http.Response, error) {
+	request, err := http.NewRequest("POST", c.Endpoint()+"rest/"+url, bytes.NewBufferString(body))
+	if err != nil {
+		return nil, err
+	}
+	return c.Do(request)
+}
+
+func checkResponse(response *http.Response) error {
 	if response.StatusCode == 404 {
-		die("Invalid endpoint or API call")
-	} else if response.StatusCode == 401 {
-		die("Invalid username or password")
+		return fmt.Errorf("Invalid endpoint or API call")
 	} else if response.StatusCode == 403 {
-		if client.apikey == "" {
-			die("Invalid CSRF token")
-		}
-		die("Invalid API key")
+		return fmt.Errorf("Invalid API key")
 	} else if response.StatusCode != 200 {
-		body := strings.TrimSpace(string(responseToBArray(response)))
-		if body != "" {
-			die(body)
+		data, err := responseToBArray(response)
+		if err != nil {
+			return err
 		}
-		die("Unknown HTTP status returned: " + response.Status)
+		body := strings.TrimSpace(string(data))
+		return fmt.Errorf("Unexpected HTTP status returned: %s\n%s", response.Status, body)
 	}
-	return response
-}
-
-func httpGet(c *cli.Context, url string) *http.Response {
-	client := getClient(c)
-	request, err := http.NewRequest("GET", client.endpoint+"/rest/"+url, nil)
-	die(err)
-	return client.handleRequest(request)
-}
-
-func httpPost(c *cli.Context, url string, body string) *http.Response {
-	client := getClient(c)
-	request, err := http.NewRequest("POST", client.endpoint+"/rest/"+url, bytes.NewBufferString(body))
-	die(err)
-	return client.handleRequest(request)
+	return nil
 }

cmd/stcli/cmd_devices.go

@@ -1,188 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-	"github.com/syncthing/syncthing/lib/config"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "devices",
-		HideHelp: true,
-		Usage:    "Device command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "list",
-				Usage:    "List registered devices",
-				Requires: &cli.Requires{},
-				Action:   devicesList,
-			},
-			{
-				Name:     "add",
-				Usage:    "Add a new device",
-				Requires: &cli.Requires{"device id", "device name?"},
-				Action:   devicesAdd,
-			},
-			{
-				Name:     "remove",
-				Usage:    "Remove an existing device",
-				Requires: &cli.Requires{"device id"},
-				Action:   devicesRemove,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a property of a device",
-				Requires: &cli.Requires{"device id", "property"},
-				Action:   devicesGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a property of a device",
-				Requires: &cli.Requires{"device id", "property", "value..."},
-				Action:   devicesSet,
-			},
-		},
-	})
-}
-
-func devicesList(c *cli.Context) {
-	cfg := getConfig(c)
-	first := true
-	writer := newTableWriter()
-	for _, device := range cfg.Devices {
-		if !first {
-			fmt.Fprintln(writer)
-		}
-		fmt.Fprintln(writer, "ID:\t", device.DeviceID, "\t")
-		fmt.Fprintln(writer, "Name:\t", device.Name, "\t(name)")
-		fmt.Fprintln(writer, "Address:\t", strings.Join(device.Addresses, " "), "\t(address)")
-		fmt.Fprintln(writer, "Compression:\t", device.Compression, "\t(compression)")
-		fmt.Fprintln(writer, "Certificate name:\t", device.CertName, "\t(certname)")
-		fmt.Fprintln(writer, "Introducer:\t", device.Introducer, "\t(introducer)")
-		first = false
-	}
-	writer.Flush()
-}
-
-func devicesAdd(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-
-	newDevice := config.DeviceConfiguration{
-		DeviceID:  id,
-		Name:      nid,
-		Addresses: []string{"dynamic"},
-	}
-
-	if len(c.Args()) > 1 {
-		newDevice.Name = c.Args()[1]
-	}
-
-	if len(c.Args()) > 2 {
-		addresses := c.Args()[2:]
-		for _, item := range addresses {
-			if item == "dynamic" {
-				continue
-			}
-			validAddress(item)
-		}
-		newDevice.Addresses = addresses
-	}
-
-	cfg := getConfig(c)
-	for _, device := range cfg.Devices {
-		if device.DeviceID == id {
-			die("Device " + nid + " already exists")
-		}
-	}
-	cfg.Devices = append(cfg.Devices, newDevice)
-	setConfig(c, cfg)
-}
-
-func devicesRemove(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-	if nid == getMyID(c) {
-		die("Cannot remove yourself")
-	}
-	cfg := getConfig(c)
-	for i, device := range cfg.Devices {
-		if device.DeviceID == id {
-			last := len(cfg.Devices) - 1
-			cfg.Devices[i] = cfg.Devices[last]
-			cfg.Devices = cfg.Devices[:last]
-			setConfig(c, cfg)
-			return
-		}
-	}
-	die("Device " + nid + " not found")
-}
-
-func devicesGet(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-	arg := c.Args()[1]
-	cfg := getConfig(c)
-	for _, device := range cfg.Devices {
-		if device.DeviceID != id {
-			continue
-		}
-		switch strings.ToLower(arg) {
-		case "name":
-			fmt.Println(device.Name)
-		case "address":
-			fmt.Println(strings.Join(device.Addresses, "\n"))
-		case "compression":
-			fmt.Println(device.Compression.String())
-		case "certname":
-			fmt.Println(device.CertName)
-		case "introducer":
-			fmt.Println(device.Introducer)
-		default:
-			die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
-		}
-		return
-	}
-	die("Device " + nid + " not found")
-}
-
-func devicesSet(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-	arg := c.Args()[1]
-	config := getConfig(c)
-	for i, device := range config.Devices {
-		if device.DeviceID != id {
-			continue
-		}
-		switch strings.ToLower(arg) {
-		case "name":
-			config.Devices[i].Name = strings.Join(c.Args()[2:], " ")
-		case "address":
-			for _, item := range c.Args()[2:] {
-				if item == "dynamic" {
-					continue
-				}
-				validAddress(item)
-			}
-			config.Devices[i].Addresses = c.Args()[2:]
-		case "compression":
-			err := config.Devices[i].Compression.UnmarshalText([]byte(c.Args()[2]))
-			die(err)
-		case "certname":
-			config.Devices[i].CertName = strings.Join(c.Args()[2:], " ")
-		case "introducer":
-			config.Devices[i].Introducer = parseBool(c.Args()[2])
-		default:
-			die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
-		}
-		setConfig(c, config)
-		return
-	}
-	die("Device " + nid + " not found")
-}

cmd/stcli/cmd_errors.go

@@ -1,67 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "errors",
-		HideHelp: true,
-		Usage:    "Error command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "show",
-				Usage:    "Show pending errors",
-				Requires: &cli.Requires{},
-				Action:   errorsShow,
-			},
-			{
-				Name:     "push",
-				Usage:    "Push an error to active clients",
-				Requires: &cli.Requires{"error message..."},
-				Action:   errorsPush,
-			},
-			{
-				Name:     "clear",
-				Usage:    "Clear pending errors",
-				Requires: &cli.Requires{},
-				Action:   wrappedHTTPPost("system/error/clear"),
-			},
-		},
-	})
-}
-
-func errorsShow(c *cli.Context) {
-	response := httpGet(c, "system/error")
-	var data map[string][]map[string]interface{}
-	json.Unmarshal(responseToBArray(response), &data)
-	writer := newTableWriter()
-	for _, item := range data["errors"] {
-		time := item["when"].(string)[:19]
-		time = strings.Replace(time, "T", " ", 1)
-		err := item["message"].(string)
-		err = strings.TrimSpace(err)
-		fmt.Fprintln(writer, time+":\t"+err)
-	}
-	writer.Flush()
-}
-
-func errorsPush(c *cli.Context) {
-	err := strings.Join(c.Args(), " ")
-	response := httpPost(c, "system/error", strings.TrimSpace(err))
-	if response.StatusCode != 200 {
-		err = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
-		body := string(responseToBArray(response))
-		if body != "" {
-			err += "\nBody: " + body
-		}
-		die(err)
-	}
-}

cmd/stcli/cmd_folders.go

@@ -1,361 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"path/filepath"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/fs"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "folders",
-		HideHelp: true,
-		Usage:    "Folder command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "list",
-				Usage:    "List available folders",
-				Requires: &cli.Requires{},
-				Action:   foldersList,
-			},
-			{
-				Name:     "add",
-				Usage:    "Add a new folder",
-				Requires: &cli.Requires{"folder id", "directory"},
-				Action:   foldersAdd,
-			},
-			{
-				Name:     "remove",
-				Usage:    "Remove an existing folder",
-				Requires: &cli.Requires{"folder id"},
-				Action:   foldersRemove,
-			},
-			{
-				Name:     "override",
-				Usage:    "Override changes from other nodes for a master folder",
-				Requires: &cli.Requires{"folder id"},
-				Action:   foldersOverride,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a property of a folder",
-				Requires: &cli.Requires{"folder id", "property"},
-				Action:   foldersGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a property of a folder",
-				Requires: &cli.Requires{"folder id", "property", "value..."},
-				Action:   foldersSet,
-			},
-			{
-				Name:     "unset",
-				Usage:    "Unset a property of a folder",
-				Requires: &cli.Requires{"folder id", "property"},
-				Action:   foldersUnset,
-			},
-			{
-				Name:     "devices",
-				Usage:    "Folder devices command group",
-				HideHelp: true,
-				Subcommands: []cli.Command{
-					{
-						Name:     "list",
-						Usage:    "List of devices which the folder is shared with",
-						Requires: &cli.Requires{"folder id"},
-						Action:   foldersDevicesList,
-					},
-					{
-						Name:     "add",
-						Usage:    "Share a folder with a device",
-						Requires: &cli.Requires{"folder id", "device id"},
-						Action:   foldersDevicesAdd,
-					},
-					{
-						Name:     "remove",
-						Usage:    "Unshare a folder with a device",
-						Requires: &cli.Requires{"folder id", "device id"},
-						Action:   foldersDevicesRemove,
-					},
-					{
-						Name:     "clear",
-						Usage:    "Unshare a folder with all devices",
-						Requires: &cli.Requires{"folder id"},
-						Action:   foldersDevicesClear,
-					},
-				},
-			},
-		},
-	})
-}
-
-func foldersList(c *cli.Context) {
-	cfg := getConfig(c)
-	first := true
-	writer := newTableWriter()
-	for _, folder := range cfg.Folders {
-		if !first {
-			fmt.Fprintln(writer)
-		}
-		fs := folder.Filesystem()
-		fmt.Fprintln(writer, "ID:\t", folder.ID, "\t")
-		fmt.Fprintln(writer, "Path:\t", fs.URI(), "\t(directory)")
-		fmt.Fprintln(writer, "Path type:\t", fs.Type(), "\t(directory-type)")
-		fmt.Fprintln(writer, "Folder type:\t", folder.Type, "\t(type)")
-		fmt.Fprintln(writer, "Ignore permissions:\t", folder.IgnorePerms, "\t(permissions)")
-		fmt.Fprintln(writer, "Rescan interval in seconds:\t", folder.RescanIntervalS, "\t(rescan)")
-
-		if folder.Versioning.Type != "" {
-			fmt.Fprintln(writer, "Versioning:\t", folder.Versioning.Type, "\t(versioning)")
-			for key, value := range folder.Versioning.Params {
-				fmt.Fprintf(writer, "Versioning %s:\t %s \t(versioning-%s)\n", key, value, key)
-			}
-		}
-		first = false
-	}
-	writer.Flush()
-}
-
-func foldersAdd(c *cli.Context) {
-	cfg := getConfig(c)
-	abs, err := filepath.Abs(c.Args()[1])
-	die(err)
-	folder := config.FolderConfiguration{
-		ID:             c.Args()[0],
-		Path:           filepath.Clean(abs),
-		FilesystemType: fs.FilesystemTypeBasic,
-	}
-	cfg.Folders = append(cfg.Folders, folder)
-	setConfig(c, cfg)
-}
-
-func foldersRemove(c *cli.Context) {
-	cfg := getConfig(c)
-	rid := c.Args()[0]
-	for i, folder := range cfg.Folders {
-		if folder.ID == rid {
-			last := len(cfg.Folders) - 1
-			cfg.Folders[i] = cfg.Folders[last]
-			cfg.Folders = cfg.Folders[:last]
-			setConfig(c, cfg)
-			return
-		}
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersOverride(c *cli.Context) {
-	cfg := getConfig(c)
-	rid := c.Args()[0]
-	for _, folder := range cfg.Folders {
-		if folder.ID == rid && folder.Type == config.FolderTypeSendOnly {
-			response := httpPost(c, "db/override", "")
-			if response.StatusCode != 200 {
-				err := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
-				body := string(responseToBArray(response))
-				if body != "" {
-					err += "\nBody: " + body
-				}
-				die(err)
-			}
-			return
-		}
-	}
-	die("Folder " + rid + " not found or folder not master")
-}
-
-func foldersGet(c *cli.Context) {
-	cfg := getConfig(c)
-	rid := c.Args()[0]
-	arg := strings.ToLower(c.Args()[1])
-	for _, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		if strings.HasPrefix(arg, "versioning-") {
-			arg = arg[11:]
-			value, ok := folder.Versioning.Params[arg]
-			if ok {
-				fmt.Println(value)
-				return
-			}
-			die("Versioning property " + c.Args()[1][11:] + " not found")
-		}
-		switch arg {
-		case "directory":
-			fmt.Println(folder.Filesystem().URI())
-		case "directory-type":
-			fmt.Println(folder.Filesystem().Type())
-		case "type":
-			fmt.Println(folder.Type)
-		case "permissions":
-			fmt.Println(folder.IgnorePerms)
-		case "rescan":
-			fmt.Println(folder.RescanIntervalS)
-		case "versioning":
-			if folder.Versioning.Type != "" {
-				fmt.Println(folder.Versioning.Type)
-			}
-		default:
-			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, directory-type, type, permissions, versioning, versioning-<key>")
-		}
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersSet(c *cli.Context) {
-	rid := c.Args()[0]
-	arg := strings.ToLower(c.Args()[1])
-	val := strings.Join(c.Args()[2:], " ")
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		if strings.HasPrefix(arg, "versioning-") {
-			cfg.Folders[i].Versioning.Params[arg[11:]] = val
-			setConfig(c, cfg)
-			return
-		}
-		switch arg {
-		case "directory":
-			cfg.Folders[i].Path = val
-		case "directory-type":
-			var fsType fs.FilesystemType
-			fsType.UnmarshalText([]byte(val))
-			cfg.Folders[i].FilesystemType = fsType
-		case "type":
-			var t config.FolderType
-			if err := t.UnmarshalText([]byte(val)); err != nil {
-				die("Invalid folder type: " + err.Error())
-			}
-			cfg.Folders[i].Type = t
-		case "permissions":
-			cfg.Folders[i].IgnorePerms = parseBool(val)
-		case "rescan":
-			cfg.Folders[i].RescanIntervalS = parseInt(val)
-		case "versioning":
-			cfg.Folders[i].Versioning.Type = val
-		default:
-			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, master, permissions, versioning, versioning-<key>")
-		}
-		setConfig(c, cfg)
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersUnset(c *cli.Context) {
-	rid := c.Args()[0]
-	arg := strings.ToLower(c.Args()[1])
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		if strings.HasPrefix(arg, "versioning-") {
-			arg = arg[11:]
-			if _, ok := folder.Versioning.Params[arg]; ok {
-				delete(cfg.Folders[i].Versioning.Params, arg)
-				setConfig(c, cfg)
-				return
-			}
-			die("Versioning property " + c.Args()[1][11:] + " not found")
-		}
-		switch arg {
-		case "versioning":
-			cfg.Folders[i].Versioning.Type = ""
-			cfg.Folders[i].Versioning.Params = make(map[string]string)
-		default:
-			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: versioning, versioning-<key>")
-		}
-		setConfig(c, cfg)
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesList(c *cli.Context) {
-	rid := c.Args()[0]
-	cfg := getConfig(c)
-	for _, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		for _, device := range folder.Devices {
-			fmt.Println(device.DeviceID)
-		}
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesAdd(c *cli.Context) {
-	rid := c.Args()[0]
-	nid := parseDeviceID(c.Args()[1])
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		for _, device := range folder.Devices {
-			if device.DeviceID == nid {
-				die("Device " + c.Args()[1] + " is already part of this folder")
-			}
-		}
-		for _, device := range cfg.Devices {
-			if device.DeviceID == nid {
-				cfg.Folders[i].Devices = append(folder.Devices, config.FolderDeviceConfiguration{
-					DeviceID: device.DeviceID,
-				})
-				setConfig(c, cfg)
-				return
-			}
-		}
-		die("Device " + c.Args()[1] + " not found in device list")
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesRemove(c *cli.Context) {
-	rid := c.Args()[0]
-	nid := parseDeviceID(c.Args()[1])
-	cfg := getConfig(c)
-	for ri, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		for ni, device := range folder.Devices {
-			if device.DeviceID == nid {
-				last := len(folder.Devices) - 1
-				cfg.Folders[ri].Devices[ni] = folder.Devices[last]
-				cfg.Folders[ri].Devices = cfg.Folders[ri].Devices[:last]
-				setConfig(c, cfg)
-				return
-			}
-		}
-		die("Device " + c.Args()[1] + " not found")
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesClear(c *cli.Context) {
-	rid := c.Args()[0]
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		cfg.Folders[i].Devices = []config.FolderDeviceConfiguration{}
-		setConfig(c, cfg)
-		return
-	}
-	die("Folder " + rid + " not found")
-}

cmd/stcli/cmd_general.go

@@ -1,94 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, []cli.Command{
-		{
-			Name:     "id",
-			Usage:    "Get ID of the Syncthing client",
-			Requires: &cli.Requires{},
-			Action:   generalID,
-		},
-		{
-			Name:     "status",
-			Usage:    "Configuration status, whether or not a restart is required for changes to take effect",
-			Requires: &cli.Requires{},
-			Action:   generalStatus,
-		},
-		{
-			Name:     "config",
-			Usage:    "Configuration",
-			Requires: &cli.Requires{},
-			Action:   generalConfiguration,
-		},
-		{
-			Name:     "restart",
-			Usage:    "Restart syncthing",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/restart"),
-		},
-		{
-			Name:     "shutdown",
-			Usage:    "Shutdown syncthing",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/shutdown"),
-		},
-		{
-			Name:     "reset",
-			Usage:    "Reset syncthing deleting all folders and devices",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/reset"),
-		},
-		{
-			Name:     "upgrade",
-			Usage:    "Upgrade syncthing (if a newer version is available)",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/upgrade"),
-		},
-		{
-			Name:     "version",
-			Usage:    "Syncthing client version",
-			Requires: &cli.Requires{},
-			Action:   generalVersion,
-		},
-	}...)
-}
-
-func generalID(c *cli.Context) {
-	fmt.Println(getMyID(c))
-}
-
-func generalStatus(c *cli.Context) {
-	response := httpGet(c, "system/config/insync")
-	var status struct{ ConfigInSync bool }
-	json.Unmarshal(responseToBArray(response), &status)
-	if !status.ConfigInSync {
-		die("Config out of sync")
-	}
-	fmt.Println("Config in sync")
-}
-
-func generalConfiguration(c *cli.Context) {
-	response := httpGet(c, "system/config")
-	var jsResponse interface{}
-	json.Unmarshal(responseToBArray(response), &jsResponse)
-	enc := json.NewEncoder(os.Stdout)
-	enc.SetIndent("", "  ")
-	enc.Encode(jsResponse)
-}
-
-func generalVersion(c *cli.Context) {
-	response := httpGet(c, "system/version")
-	version := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &version)
-	prettyPrintJSON(version)
-}

cmd/stcli/cmd_gui.go

@@ -1,127 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "gui",
-		HideHelp: true,
-		Usage:    "GUI command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "dump",
-				Usage:    "Show all GUI configuration settings",
-				Requires: &cli.Requires{},
-				Action:   guiDump,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a GUI configuration setting",
-				Requires: &cli.Requires{"setting"},
-				Action:   guiGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a GUI configuration setting",
-				Requires: &cli.Requires{"setting", "value"},
-				Action:   guiSet,
-			},
-			{
-				Name:     "unset",
-				Usage:    "Unset a GUI configuration setting",
-				Requires: &cli.Requires{"setting"},
-				Action:   guiUnset,
-			},
-		},
-	})
-}
-
-func guiDump(c *cli.Context) {
-	cfg := getConfig(c).GUI
-	writer := newTableWriter()
-	fmt.Fprintln(writer, "Enabled:\t", cfg.Enabled, "\t(enabled)")
-	fmt.Fprintln(writer, "Use HTTPS:\t", cfg.UseTLS(), "\t(tls)")
-	fmt.Fprintln(writer, "Listen Addresses:\t", cfg.Address(), "\t(address)")
-	if cfg.User != "" {
-		fmt.Fprintln(writer, "Authentication User:\t", cfg.User, "\t(username)")
-		fmt.Fprintln(writer, "Authentication Password:\t", cfg.Password, "\t(password)")
-	}
-	if cfg.APIKey != "" {
-		fmt.Fprintln(writer, "API Key:\t", cfg.APIKey, "\t(apikey)")
-	}
-	writer.Flush()
-}
-
-func guiGet(c *cli.Context) {
-	cfg := getConfig(c).GUI
-	arg := c.Args()[0]
-	switch strings.ToLower(arg) {
-	case "enabled":
-		fmt.Println(cfg.Enabled)
-	case "tls":
-		fmt.Println(cfg.UseTLS())
-	case "address":
-		fmt.Println(cfg.Address())
-	case "user":
-		if cfg.User != "" {
-			fmt.Println(cfg.User)
-		}
-	case "password":
-		if cfg.User != "" {
-			fmt.Println(cfg.Password)
-		}
-	case "apikey":
-		if cfg.APIKey != "" {
-			fmt.Println(cfg.APIKey)
-		}
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
-	}
-}
-
-func guiSet(c *cli.Context) {
-	cfg := getConfig(c)
-	arg := c.Args()[0]
-	val := c.Args()[1]
-	switch strings.ToLower(arg) {
-	case "enabled":
-		cfg.GUI.Enabled = parseBool(val)
-	case "tls":
-		cfg.GUI.RawUseTLS = parseBool(val)
-	case "address":
-		validAddress(val)
-		cfg.GUI.RawAddress = val
-	case "user":
-		cfg.GUI.User = val
-	case "password":
-		cfg.GUI.Password = val
-	case "apikey":
-		cfg.GUI.APIKey = val
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
-	}
-	setConfig(c, cfg)
-}
-
-func guiUnset(c *cli.Context) {
-	cfg := getConfig(c)
-	arg := c.Args()[0]
-	switch strings.ToLower(arg) {
-	case "user":
-		cfg.GUI.User = ""
-	case "password":
-		cfg.GUI.Password = ""
-	case "apikey":
-		cfg.GUI.APIKey = ""
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: user, password, apikey")
-	}
-	setConfig(c, cfg)
-}

cmd/stcli/cmd_options.go

@@ -1,173 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "options",
-		HideHelp: true,
-		Usage:    "Options command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "dump",
-				Usage:    "Show all Syncthing option settings",
-				Requires: &cli.Requires{},
-				Action:   optionsDump,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a Syncthing option setting",
-				Requires: &cli.Requires{"setting"},
-				Action:   optionsGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a Syncthing option setting",
-				Requires: &cli.Requires{"setting", "value..."},
-				Action:   optionsSet,
-			},
-		},
-	})
-}
-
-func optionsDump(c *cli.Context) {
-	cfg := getConfig(c).Options
-	writer := newTableWriter()
-
-	fmt.Fprintln(writer, "Sync protocol listen addresses:\t", strings.Join(cfg.ListenAddresses, " "), "\t(addresses)")
-	fmt.Fprintln(writer, "Global discovery enabled:\t", cfg.GlobalAnnEnabled, "\t(globalannenabled)")
-	fmt.Fprintln(writer, "Global discovery servers:\t", strings.Join(cfg.GlobalAnnServers, " "), "\t(globalannserver)")
-
-	fmt.Fprintln(writer, "Local discovery enabled:\t", cfg.LocalAnnEnabled, "\t(localannenabled)")
-	fmt.Fprintln(writer, "Local discovery port:\t", cfg.LocalAnnPort, "\t(localannport)")
-
-	fmt.Fprintln(writer, "Outgoing rate limit in KiB/s:\t", cfg.MaxSendKbps, "\t(maxsend)")
-	fmt.Fprintln(writer, "Incoming rate limit in KiB/s:\t", cfg.MaxRecvKbps, "\t(maxrecv)")
-	fmt.Fprintln(writer, "Reconnect interval in seconds:\t", cfg.ReconnectIntervalS, "\t(reconnect)")
-	fmt.Fprintln(writer, "Start browser:\t", cfg.StartBrowser, "\t(browser)")
-	fmt.Fprintln(writer, "Enable UPnP:\t", cfg.NATEnabled, "\t(nat)")
-	fmt.Fprintln(writer, "UPnP Lease in minutes:\t", cfg.NATLeaseM, "\t(natlease)")
-	fmt.Fprintln(writer, "UPnP Renewal period in minutes:\t", cfg.NATRenewalM, "\t(natrenew)")
-	fmt.Fprintln(writer, "Restart on Wake Up:\t", cfg.RestartOnWakeup, "\t(wake)")
-
-	reporting := "unrecognized value"
-	switch cfg.URAccepted {
-	case -1:
-		reporting = "false"
-	case 0:
-		reporting = "undecided/false"
-	case 1:
-		reporting = "true"
-	}
-	fmt.Fprintln(writer, "Anonymous usage reporting:\t", reporting, "\t(reporting)")
-
-	writer.Flush()
-}
-
-func optionsGet(c *cli.Context) {
-	cfg := getConfig(c).Options
-	arg := c.Args()[0]
-	switch strings.ToLower(arg) {
-	case "address":
-		fmt.Println(strings.Join(cfg.ListenAddresses, "\n"))
-	case "globalannenabled":
-		fmt.Println(cfg.GlobalAnnEnabled)
-	case "globalannservers":
-		fmt.Println(strings.Join(cfg.GlobalAnnServers, "\n"))
-	case "localannenabled":
-		fmt.Println(cfg.LocalAnnEnabled)
-	case "localannport":
-		fmt.Println(cfg.LocalAnnPort)
-	case "maxsend":
-		fmt.Println(cfg.MaxSendKbps)
-	case "maxrecv":
-		fmt.Println(cfg.MaxRecvKbps)
-	case "reconnect":
-		fmt.Println(cfg.ReconnectIntervalS)
-	case "browser":
-		fmt.Println(cfg.StartBrowser)
-	case "nat":
-		fmt.Println(cfg.NATEnabled)
-	case "natlease":
-		fmt.Println(cfg.NATLeaseM)
-	case "natrenew":
-		fmt.Println(cfg.NATRenewalM)
-	case "reporting":
-		switch cfg.URAccepted {
-		case -1:
-			fmt.Println("false")
-		case 0:
-			fmt.Println("undecided/false")
-		case 1:
-			fmt.Println("true")
-		default:
-			fmt.Println("unknown")
-		}
-	case "wake":
-		fmt.Println(cfg.RestartOnWakeup)
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
-	}
-}
-
-func optionsSet(c *cli.Context) {
-	config := getConfig(c)
-	arg := c.Args()[0]
-	val := c.Args()[1]
-	switch strings.ToLower(arg) {
-	case "address":
-		for _, item := range c.Args().Tail() {
-			validAddress(item)
-		}
-		config.Options.ListenAddresses = c.Args().Tail()
-	case "globalannenabled":
-		config.Options.GlobalAnnEnabled = parseBool(val)
-	case "globalannserver":
-		for _, item := range c.Args().Tail() {
-			validAddress(item)
-		}
-		config.Options.GlobalAnnServers = c.Args().Tail()
-	case "localannenabled":
-		config.Options.LocalAnnEnabled = parseBool(val)
-	case "localannport":
-		config.Options.LocalAnnPort = parsePort(val)
-	case "maxsend":
-		config.Options.MaxSendKbps = parseUint(val)
-	case "maxrecv":
-		config.Options.MaxRecvKbps = parseUint(val)
-	case "reconnect":
-		config.Options.ReconnectIntervalS = parseUint(val)
-	case "browser":
-		config.Options.StartBrowser = parseBool(val)
-	case "nat":
-		config.Options.NATEnabled = parseBool(val)
-	case "natlease":
-		config.Options.NATLeaseM = parseUint(val)
-	case "natrenew":
-		config.Options.NATRenewalM = parseUint(val)
-	case "reporting":
-		switch strings.ToLower(val) {
-		case "u", "undecided", "unset":
-			config.Options.URAccepted = 0
-		default:
-			boolvalue := parseBool(val)
-			if boolvalue {
-				config.Options.URAccepted = 1
-			} else {
-				config.Options.URAccepted = -1
-			}
-		}
-	case "wake":
-		config.Options.RestartOnWakeup = parseBool(val)
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
-	}
-	setConfig(c, config)
-}

cmd/stcli/cmd_report.go

@@ -1,72 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "report",
-		HideHelp: true,
-		Usage:    "Reporting command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "system",
-				Usage:    "Report system state",
-				Requires: &cli.Requires{},
-				Action:   reportSystem,
-			},
-			{
-				Name:     "connections",
-				Usage:    "Report about connections to other devices",
-				Requires: &cli.Requires{},
-				Action:   reportConnections,
-			},
-			{
-				Name:     "usage",
-				Usage:    "Usage report",
-				Requires: &cli.Requires{},
-				Action:   reportUsage,
-			},
-		},
-	})
-}
-
-func reportSystem(c *cli.Context) {
-	response := httpGet(c, "system/status")
-	data := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &data)
-	prettyPrintJSON(data)
-}
-
-func reportConnections(c *cli.Context) {
-	response := httpGet(c, "system/connections")
-	data := make(map[string]map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &data)
-	var overall map[string]interface{}
-	for key, value := range data {
-		if key == "total" {
-			overall = value
-			continue
-		}
-		value["Device ID"] = key
-		prettyPrintJSON(value)
-		fmt.Println()
-	}
-	if overall != nil {
-		fmt.Println("=== Overall statistics ===")
-		prettyPrintJSON(overall)
-	}
-}
-
-func reportUsage(c *cli.Context) {
-	response := httpGet(c, "svc/report")
-	report := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &report)
-	prettyPrintJSON(report)
-}

cmd/stcli/errors.go

@@ -0,0 +1,60 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/urfave/cli"
+)
+
+var errorsCommand = cli.Command{
+	Name:     "errors",
+	HideHelp: true,
+	Usage:    "Error command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "show",
+			Usage:  "Show pending errors",
+			Action: expects(0, dumpOutput("system/error")),
+		},
+		{
+			Name:      "push",
+			Usage:     "Push an error to active clients",
+			ArgsUsage: "[error message]",
+			Action:    expects(1, errorsPush),
+		},
+		{
+			Name:   "clear",
+			Usage:  "Clear pending errors",
+			Action: expects(0, emptyPost("system/error/clear")),
+		},
+	},
+}
+
+func errorsPush(c *cli.Context) error {
+	client := c.App.Metadata["client"].(*APIClient)
+	errStr := strings.Join(c.Args(), " ")
+	response, err := client.Post("system/error", strings.TrimSpace(errStr))
+	if err != nil {
+		return err
+	}
+	if response.StatusCode != 200 {
+		errStr = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
+		bytes, err := responseToBArray(response)
+		if err != nil {
+			return err
+		}
+		body := string(bytes)
+		if body != "" {
+			errStr += "\nBody: " + body
+		}
+		return fmt.Errorf(errStr)
+	}
+	return nil
+}

cmd/stcli/labels.go

@@ -1,31 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-var jsonAttributeLabels = map[string]string{
-	"folderMaxMiB":   "Largest folder size in MiB",
-	"folderMaxFiles": "Largest folder file count",
-	"longVersion":    "Long version",
-	"totMiB":         "Total size in MiB",
-	"totFiles":       "Total files",
-	"uniqueID":       "Unique ID",
-	"numFolders":     "Folder count",
-	"numDevices":     "Device count",
-	"memoryUsageMiB": "Memory usage in MiB",
-	"memorySize":     "Total memory in MiB",
-	"sha256Perf":     "SHA256 Benchmark",
-	"At":             "Last contacted",
-	"Completion":     "Percent complete",
-	"InBytesTotal":   "Total bytes received",
-	"OutBytesTotal":  "Total bytes sent",
-	"ClientVersion":  "Client version",
-	"alloc":          "Memory allocated in bytes",
-	"sys":            "Memory using in bytes",
-	"cpuPercent":     "CPU load in percent",
-	"extAnnounceOK":  "External announcments working",
-	"goroutines":     "Number of Go routines",
-	"myID":           "Client ID",
-	"tilde":          "Tilde expands to",
-	"arch":           "Architecture",
-	"os":             "OS",
-}

cmd/stcli/main.go

@@ -1,63 +1,192 @@
-// Copyright (C) 2014 Audrius Butkevičius
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
-	"sort"
+	"bufio"
+	"crypto/tls"
+	"encoding/json"
+	"flag"
+	"log"
+	"os"
+	"reflect"
 
-	"github.com/AudriusButkevicius/cli"
+	"github.com/AudriusButkevicius/recli"
+	"github.com/flynn-archive/go-shlex"
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/locations"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/urfave/cli"
 )
 
-type ByAlphabet []cli.Command
-
-func (a ByAlphabet) Len() int           { return len(a) }
-func (a ByAlphabet) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
-func (a ByAlphabet) Less(i, j int) bool { return a[i].Name < a[j].Name }
-
-var cliCommands []cli.Command
-
 func main() {
-	app := cli.NewApp()
-	app.Name = "syncthing-cli"
-	app.Author = "Audrius Butkevičius"
-	app.Email = "audrius.butkevicius@gmail.com"
-	app.Usage = "Syncthing command line interface"
-	app.Version = "0.1"
-	app.HideHelp = true
+	// This is somewhat a hack around a chicken and egg problem.
+	// We need to set the home directory and potentially other flags to know where the syncthing instance is running
+	// in order to get it's config ... which we then use to construct the actual CLI ... at which point it's too late
+	// to add flags there...
+	homeBaseDir := locations.GetBaseDir(locations.ConfigBaseDir)
+	guiCfg := config.GUIConfiguration{}
 
-	app.Flags = []cli.Flag{
+	flags := flag.NewFlagSet("", flag.ContinueOnError)
+	flags.StringVar(&guiCfg.RawAddress, "gui-address", guiCfg.RawAddress, "Override GUI address (e.g. \"http://192.0.2.42:8443\")")
+	flags.StringVar(&guiCfg.APIKey, "gui-apikey", guiCfg.APIKey, "Override GUI API key")
+	flags.StringVar(&homeBaseDir, "home", homeBaseDir, "Set configuration directory")
+
+	// Implement the same flags at the lower CLI, with the same default values (pre-parse), but do nothing with them.
+	// This is so that we could reuse os.Args
+	fakeFlags := []cli.Flag{
 		cli.StringFlag{
-			Name:   "endpoint, e",
-			Value:  "http://127.0.0.1:8384",
-			Usage:  "End point to connect to",
-			EnvVar: "STENDPOINT",
+			Name:  "gui-address",
+			Value: guiCfg.RawAddress,
+			Usage: "Override GUI address (e.g. \"http://192.0.2.42:8443\")",
 		},
 		cli.StringFlag{
-			Name:   "apikey, k",
-			Value:  "",
-			Usage:  "API Key",
-			EnvVar: "STAPIKEY",
+			Name:  "gui-apikey",
+			Value: guiCfg.APIKey,
+			Usage: "Override GUI API key",
 		},
 		cli.StringFlag{
-			Name:   "username, u",
-			Value:  "",
-			Usage:  "Username",
-			EnvVar: "STUSERNAME",
-		},
-		cli.StringFlag{
-			Name:   "password, p",
-			Value:  "",
-			Usage:  "Password",
-			EnvVar: "STPASSWORD",
-		},
-		cli.BoolFlag{
-			Name:   "insecure, i",
-			Usage:  "Do not verify SSL certificate",
-			EnvVar: "STINSECURE",
+			Name:  "home",
+			Value: homeBaseDir,
+			Usage: "Set configuration directory",
 		},
 	}
 
-	sort.Sort(ByAlphabet(cliCommands))
-	app.Commands = cliCommands
-	app.RunAndExitOnError()
+	// Do not print usage of these flags, and ignore errors as this can't understand plenty of things
+	flags.Usage = func() {}
+	_ = flags.Parse(os.Args[1:])
+
+	// Now if the API key and address is not provided (we are not connecting to a remote instance),
+	// try to rip it out of the config.
+	if guiCfg.RawAddress == "" && guiCfg.APIKey == "" {
+		// Update the base directory
+		err := locations.SetBaseDir(locations.ConfigBaseDir, homeBaseDir)
+		if err != nil {
+			log.Fatal(errors.Wrap(err, "setting home"))
+		}
+
+		// Load the certs and get the ID
+		cert, err := tls.LoadX509KeyPair(
+			locations.Get(locations.CertFile),
+			locations.Get(locations.KeyFile),
+		)
+		if err != nil {
+			log.Fatal(errors.Wrap(err, "reading device ID"))
+		}
+
+		myID := protocol.NewDeviceID(cert.Certificate[0])
+
+		// Load the config
+		cfg, err := config.Load(locations.Get(locations.ConfigFile), myID, events.NoopLogger)
+		if err != nil {
+			log.Fatalln(errors.Wrap(err, "loading config"))
+		}
+
+		guiCfg = cfg.GUI()
+	} else if guiCfg.Address() == "" || guiCfg.APIKey == "" {
+		log.Fatalln("Both -gui-address and -gui-apikey should be specified")
+	}
+
+	if guiCfg.Address() == "" {
+		log.Fatalln("Could not find GUI Address")
+	}
+
+	if guiCfg.APIKey == "" {
+		log.Fatalln("Could not find GUI API key")
+	}
+
+	client := getClient(guiCfg)
+
+	cfg, err := getConfig(client)
+	original := cfg.Copy()
+	if err != nil {
+		log.Fatalln(errors.Wrap(err, "getting config"))
+	}
+
+	// Copy the config and set the default flags
+	recliCfg := recli.DefaultConfig
+	recliCfg.IDTag.Name = "xml"
+	recliCfg.SkipTag.Name = "json"
+
+	commands, err := recli.New(recliCfg).Construct(&cfg)
+	if err != nil {
+		log.Fatalln(errors.Wrap(err, "config reflect"))
+	}
+
+	// Construct the actual CLI
+	app := cli.NewApp()
+	app.Name = "stcli"
+	app.HelpName = app.Name
+	app.Author = "The Syncthing Authors"
+	app.Usage = "Syncthing command line interface"
+	app.Version = build.Version
+	app.Flags = fakeFlags
+	app.Metadata = map[string]interface{}{
+		"client": client,
+	}
+	app.Commands = []cli.Command{
+		{
+			Name:        "config",
+			HideHelp:    true,
+			Usage:       "Configuration modification command group",
+			Subcommands: commands,
+		},
+		showCommand,
+		operationCommand,
+		errorsCommand,
+	}
+
+	tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
+	if !tty {
+		// Not a TTY, consume from stdin
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			input, err := shlex.Split(scanner.Text())
+			if err != nil {
+				log.Fatalln(errors.Wrap(err, "parsing input"))
+			}
+			if len(input) == 0 {
+				continue
+			}
+			err = app.Run(append(os.Args, input...))
+			if err != nil {
+				log.Fatalln(err)
+			}
+		}
+		err = scanner.Err()
+		if err != nil {
+			log.Fatalln(err)
+		}
+	} else {
+		err = app.Run(os.Args)
+		if err != nil {
+			log.Fatalln(err)
+		}
+	}
+
+	if !reflect.DeepEqual(cfg, original) {
+		body, err := json.MarshalIndent(cfg, "", "  ")
+		if err != nil {
+			log.Fatalln(err)
+		}
+		resp, err := client.Post("system/config", string(body))
+		if err != nil {
+			log.Fatalln(err)
+		}
+		if resp.StatusCode != 200 {
+			body, err := responseToBArray(resp)
+			if err != nil {
+				log.Fatalln(err)
+			}
+			log.Fatalln(string(body))
+		}
+	}
 }

cmd/stcli/operations.go

@@ -0,0 +1,78 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli"
+)
+
+var operationCommand = cli.Command{
+	Name:     "operations",
+	HideHelp: true,
+	Usage:    "Operation command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "restart",
+			Usage:  "Restart syncthing",
+			Action: expects(0, emptyPost("system/restart")),
+		},
+		{
+			Name:   "shutdown",
+			Usage:  "Shutdown syncthing",
+			Action: expects(0, emptyPost("system/shutdown")),
+		},
+		{
+			Name:   "reset",
+			Usage:  "Reset syncthing deleting all folders and devices",
+			Action: expects(0, emptyPost("system/reset")),
+		},
+		{
+			Name:   "upgrade",
+			Usage:  "Upgrade syncthing (if a newer version is available)",
+			Action: expects(0, emptyPost("system/upgrade")),
+		},
+		{
+			Name:      "folder-override",
+			Usage:     "Override changes on folder (remote for sendonly, local for receiveonly)",
+			ArgsUsage: "[folder id]",
+			Action:    expects(1, foldersOverride),
+		},
+	},
+}
+
+func foldersOverride(c *cli.Context) error {
+	client := c.App.Metadata["client"].(*APIClient)
+	cfg, err := getConfig(client)
+	if err != nil {
+		return err
+	}
+	rid := c.Args()[0]
+	for _, folder := range cfg.Folders {
+		if folder.ID == rid {
+			response, err := client.Post("db/override", "")
+			if err != nil {
+				return err
+			}
+			if response.StatusCode != 200 {
+				errStr := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
+				bytes, err := responseToBArray(response)
+				if err != nil {
+					return err
+				}
+				body := string(bytes)
+				if body != "" {
+					errStr += "\nBody: " + body
+				}
+				return fmt.Errorf(errStr)
+			}
+			return nil
+		}
+	}
+	return fmt.Errorf("Folder " + rid + " not found")
+}

cmd/stcli/show.go

@@ -0,0 +1,44 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"github.com/urfave/cli"
+)
+
+var showCommand = cli.Command{
+	Name:     "show",
+	HideHelp: true,
+	Usage:    "Show command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "version",
+			Usage:  "Show syncthing client version",
+			Action: expects(0, dumpOutput("system/version")),
+		},
+		{
+			Name:   "config-status",
+			Usage:  "Show configuration status, whether or not a restart is required for changes to take effect",
+			Action: expects(0, dumpOutput("system/config/insync")),
+		},
+		{
+			Name:   "system",
+			Usage:  "Show system status",
+			Action: expects(0, dumpOutput("system/status")),
+		},
+		{
+			Name:   "connections",
+			Usage:  "Report about connections to other devices",
+			Action: expects(0, dumpOutput("system/connections")),
+		},
+		{
+			Name:   "usage",
+			Usage:  "Show usage report",
+			Action: expects(0, dumpOutput("svc/report")),
+		},
+	},
+}

cmd/stcli/utils.go

@@ -1,4 +1,8 @@
-// Copyright (C) 2014 Audrius Butkevičius
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -8,158 +12,83 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"text/tabwriter"
-	"unicode"
 
-	"github.com/AudriusButkevicius/cli"
 	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/urfave/cli"
 )
 
-func responseToBArray(response *http.Response) []byte {
-	defer response.Body.Close()
+func responseToBArray(response *http.Response) ([]byte, error) {
 	bytes, err := ioutil.ReadAll(response.Body)
 	if err != nil {
-		die(err)
+		return nil, err
 	}
-	return bytes
+	return bytes, response.Body.Close()
 }
 
-func die(vals ...interface{}) {
-	if len(vals) > 1 || vals[0] != nil {
-		os.Stderr.WriteString(fmt.Sprintln(vals...))
-		os.Exit(1)
+func emptyPost(url string) cli.ActionFunc {
+	return func(c *cli.Context) error {
+		client := c.App.Metadata["client"].(*APIClient)
+		_, err := client.Post(url, "")
+		return err
 	}
 }
 
-func wrappedHTTPPost(url string) func(c *cli.Context) {
-	return func(c *cli.Context) {
-		httpPost(c, url, "")
-	}
-}
-
-func prettyPrintJSON(json map[string]interface{}) {
-	writer := newTableWriter()
-	remap := make(map[string]interface{})
-	for k, v := range json {
-		key, ok := jsonAttributeLabels[k]
-		if !ok {
-			key = firstUpper(k)
+func dumpOutput(url string) cli.ActionFunc {
+	return func(c *cli.Context) error {
+		client := c.App.Metadata["client"].(*APIClient)
+		response, err := client.Get(url)
+		if err != nil {
+			return err
 		}
-		remap[key] = v
+		return prettyPrintResponse(c, response)
 	}
+}
 
-	jsonKeys := make([]string, 0, len(remap))
-	for key := range remap {
-		jsonKeys = append(jsonKeys, key)
+func getConfig(c *APIClient) (config.Configuration, error) {
+	cfg := config.Configuration{}
+	response, err := c.Get("system/config")
+	if err != nil {
+		return cfg, err
 	}
-	sort.Strings(jsonKeys)
-	for _, k := range jsonKeys {
-		var value string
-		rvalue := remap[k]
-		switch rvalue.(type) {
-		case int, int16, int32, int64, uint, uint16, uint32, uint64, float32, float64:
-			value = fmt.Sprintf("%.0f", rvalue)
-		default:
-			value = fmt.Sprint(rvalue)
+	bytes, err := responseToBArray(response)
+	if err != nil {
+		return cfg, err
+	}
+	err = json.Unmarshal(bytes, &cfg)
+	if err == nil {
+		return cfg, err
+	}
+	return cfg, nil
+}
+
+func expects(n int, actionFunc cli.ActionFunc) cli.ActionFunc {
+	return func(ctx *cli.Context) error {
+		if ctx.NArg() != n {
+			plural := ""
+			if n != 1 {
+				plural = "s"
+			}
+			return fmt.Errorf("expected %d argument%s, got %d", n, plural, ctx.NArg())
 		}
-		if value == "" {
-			continue
-		}
-		fmt.Fprintln(writer, k+":\t"+value)
-	}
-	writer.Flush()
-}
-
-func firstUpper(str string) string {
-	for i, v := range str {
-		return string(unicode.ToUpper(v)) + str[i+1:]
-	}
-	return ""
-}
-
-func newTableWriter() *tabwriter.Writer {
-	writer := new(tabwriter.Writer)
-	writer.Init(os.Stdout, 0, 8, 0, '\t', 0)
-	return writer
-}
-
-func getMyID(c *cli.Context) string {
-	response := httpGet(c, "system/status")
-	data := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &data)
-	return data["myID"].(string)
-}
-
-func getConfig(c *cli.Context) config.Configuration {
-	response := httpGet(c, "system/config")
-	config := config.Configuration{}
-	json.Unmarshal(responseToBArray(response), &config)
-	return config
-}
-
-func setConfig(c *cli.Context, cfg config.Configuration) {
-	body, err := json.Marshal(cfg)
-	die(err)
-	response := httpPost(c, "system/config", string(body))
-	if response.StatusCode != 200 {
-		die("Unexpected status code", response.StatusCode)
+		return actionFunc(ctx)
 	}
 }
 
-func parseBool(input string) bool {
-	val, err := strconv.ParseBool(input)
+func prettyPrintJSON(data interface{}) error {
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	return enc.Encode(data)
+}
+
+func prettyPrintResponse(c *cli.Context, response *http.Response) error {
+	bytes, err := responseToBArray(response)
 	if err != nil {
-		die(input + " is not a valid value for a boolean")
+		return err
 	}
-	return val
-}
-
-func parseInt(input string) int {
-	val, err := strconv.ParseInt(input, 0, 64)
-	if err != nil {
-		die(input + " is not a valid value for an integer")
-	}
-	return int(val)
-}
-
-func parseUint(input string) int {
-	val, err := strconv.ParseUint(input, 0, 64)
-	if err != nil {
-		die(input + " is not a valid value for an unsigned integer")
-	}
-	return int(val)
-}
-
-func parsePort(input string) int {
-	port := parseUint(input)
-	if port < 1 || port > 65535 {
-		die(input + " is not a valid port\nExpected value between 1 and 65535")
-	}
-	return port
-}
-
-func validAddress(input string) {
-	tokens := strings.Split(input, ":")
-	if len(tokens) != 2 {
-		die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
-	}
-	matched, err := regexp.MatchString("^[a-zA-Z0-9]+([-a-zA-Z0-9.]+[-a-zA-Z0-9]+)?$", tokens[0])
-	die(err)
-	if !matched {
-		die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
-	}
-	parsePort(tokens[1])
-}
-
-func parseDeviceID(input string) protocol.DeviceID {
-	device, err := protocol.DeviceIDFromString(input)
-	if err != nil {
-		die(input + " is not a valid device id")
-	}
-	return device
+	var data interface{}
+	if err := json.Unmarshal(bytes, &data); err != nil {
+		return err
+	}
+	// TODO: Check flag for pretty print format
+	return prettyPrintJSON(data)
 }

cmd/stcrashreceiver/sentry.go

@@ -0,0 +1,204 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"regexp"
+	"strings"
+	"sync"
+
+	raven "github.com/getsentry/raven-go"
+	"github.com/maruel/panicparse/stack"
+)
+
+const reportServer = "https://crash.syncthing.net/report/"
+
+var loader = newGithubSourceCodeLoader()
+
+func init() {
+	raven.SetSourceCodeLoader(loader)
+}
+
+var (
+	clients    = make(map[string]*raven.Client)
+	clientsMut sync.Mutex
+)
+
+func sendReport(dsn, path string, report []byte) error {
+	pkt, err := parseReport(path, report)
+	if err != nil {
+		return err
+	}
+
+	clientsMut.Lock()
+	defer clientsMut.Unlock()
+
+	cli, ok := clients[dsn]
+	if !ok {
+		cli, err = raven.New(dsn)
+		if err != nil {
+			return err
+		}
+		clients[dsn] = cli
+	}
+
+	// The client sets release and such on the packet before sending, in the
+	// misguided idea that it knows this better than than the packet we give
+	// it. So we copy the values from the packet to the client first...
+	cli.SetRelease(pkt.Release)
+	cli.SetEnvironment(pkt.Environment)
+
+	defer cli.Wait()
+	_, errC := cli.Capture(pkt, nil)
+	return <-errC
+}
+
+func parseReport(path string, report []byte) (*raven.Packet, error) {
+	parts := bytes.SplitN(report, []byte("\n"), 2)
+	if len(parts) != 2 {
+		return nil, errors.New("no first line")
+	}
+
+	version, err := parseVersion(string(parts[0]))
+	if err != nil {
+		return nil, err
+	}
+	report = parts[1]
+
+	foundPanic := false
+	var subjectLine []byte
+	for {
+		parts = bytes.SplitN(report, []byte("\n"), 2)
+		if len(parts) != 2 {
+			return nil, errors.New("no panic line found")
+		}
+
+		line := parts[0]
+		report = parts[1]
+
+		if foundPanic {
+			// The previous line was our "Panic at ..." header. We are now
+			// at the beginning of the real panic trace and this is our
+			// subject line.
+			subjectLine = line
+			break
+		} else if bytes.HasPrefix(line, []byte("Panic at")) {
+			foundPanic = true
+		}
+	}
+
+	r := bytes.NewReader(report)
+	ctx, err := stack.ParseDump(r, ioutil.Discard, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// Lock the source code loader to the version we are processing here.
+	if version.commit != "" {
+		// We have a commit hash, so we know exactly which source to use
+		loader.LockWithVersion(version.commit)
+	} else if strings.HasPrefix(version.tag, "v") {
+		// Lets hope the tag is close enough
+		loader.LockWithVersion(version.tag)
+	} else {
+		// Last resort
+		loader.LockWithVersion("master")
+	}
+	defer loader.Unlock()
+
+	var trace raven.Stacktrace
+	for _, gr := range ctx.Goroutines {
+		if gr.First {
+			trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
+			for i, sc := range gr.Stack.Calls {
+				trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name(), sc.SrcPath, sc.Line, 3, nil)
+			}
+			break
+		}
+	}
+
+	pkt := &raven.Packet{
+		Message:     string(subjectLine),
+		Platform:    "go",
+		Release:     version.tag,
+		Environment: version.environment(),
+		Tags: raven.Tags{
+			raven.Tag{Key: "version", Value: version.version},
+			raven.Tag{Key: "tag", Value: version.tag},
+			raven.Tag{Key: "codename", Value: version.codename},
+			raven.Tag{Key: "runtime", Value: version.runtime},
+			raven.Tag{Key: "goos", Value: version.goos},
+			raven.Tag{Key: "goarch", Value: version.goarch},
+			raven.Tag{Key: "builder", Value: version.builder},
+		},
+		Extra: raven.Extra{
+			"url": reportServer + path,
+		},
+		Interfaces: []raven.Interface{&trace},
+	}
+	if version.commit != "" {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
+	}
+
+	return pkt, nil
+}
+
+// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC
+var longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)`)
+
+type version struct {
+	version  string // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
+	tag      string // "v1.1.4-rc.1"
+	commit   string // "6aaae618", blank when absent
+	codename string // "Erbium Earthworm"
+	runtime  string // "go1.12.5"
+	goos     string // "darwin"
+	goarch   string // "amd64"
+	builder  string // "jb@kvin.kastelo.net"
+}
+
+func (v version) environment() string {
+	if v.commit != "" {
+		return "Development"
+	}
+	if strings.Contains(v.tag, "-rc.") {
+		return "Candidate"
+	}
+	if strings.Contains(v.tag, "-") {
+		return "Beta"
+	}
+	return "Stable"
+}
+
+func parseVersion(line string) (version, error) {
+	m := longVersionRE.FindStringSubmatch(line)
+	if len(m) == 0 {
+		return version{}, errors.New("unintelligeble version string")
+	}
+
+	v := version{
+		version:  m[1],
+		codename: m[2],
+		runtime:  m[3],
+		goos:     m[4],
+		goarch:   m[5],
+		builder:  m[6],
+	}
+	parts := strings.Split(v.version, "+")
+	v.tag = parts[0]
+	if len(parts) > 1 {
+		fields := strings.Split(parts[1], "-")
+		if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
+			v.commit = fields[1][1:]
+		}
+	}
+
+	return v, nil
+}

cmd/stcrashreceiver/sentry_test.go

@@ -0,0 +1,64 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+)
+
+func TestParseVersion(t *testing.T) {
+	cases := []struct {
+		longVersion string
+		parsed      version
+	}{
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		v, err := parseVersion(tc.longVersion)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		if v != tc.parsed {
+			t.Error(v)
+		}
+	}
+}
+
+func TestParseReport(t *testing.T) {
+	bs, err := ioutil.ReadFile("_testdata/panic.log")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pkt, err := parseReport("1/2/345", bs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bs, err = pkt.JSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Printf("%s\n", bs)
+}

cmd/stcrashreceiver/sourcecodeloader.go

@@ -0,0 +1,114 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	urlPrefix   = "https://raw.githubusercontent.com/syncthing/syncthing/"
+	httpTimeout = 10 * time.Second
+)
+
+type githubSourceCodeLoader struct {
+	mut     sync.Mutex
+	version string
+	cache   map[string]map[string][][]byte // version -> file -> lines
+	client  *http.Client
+}
+
+func newGithubSourceCodeLoader() *githubSourceCodeLoader {
+	return &githubSourceCodeLoader{
+		cache:  make(map[string]map[string][][]byte),
+		client: &http.Client{Timeout: httpTimeout},
+	}
+}
+
+func (l *githubSourceCodeLoader) LockWithVersion(version string) {
+	l.mut.Lock()
+	l.version = version
+	if _, ok := l.cache[version]; !ok {
+		l.cache[version] = make(map[string][][]byte)
+	}
+}
+
+func (l *githubSourceCodeLoader) Unlock() {
+	l.mut.Unlock()
+}
+
+func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]byte, int) {
+	filename = filepath.ToSlash(filename)
+	lines, ok := l.cache[l.version][filename]
+	if !ok {
+		// Cache whatever we managed to find (or nil if nothing, so we don't try again)
+		defer func() {
+			l.cache[l.version][filename] = lines
+		}()
+
+		knownPrefixes := []string{"/lib/", "/cmd/"}
+		var idx int
+		for _, pref := range knownPrefixes {
+			idx = strings.Index(filename, pref)
+			if idx >= 0 {
+				break
+			}
+		}
+		if idx == -1 {
+			return nil, 0
+		}
+
+		url := urlPrefix + l.version + filename[idx:]
+		resp, err := l.client.Get(url)
+
+		if err != nil || resp.StatusCode != http.StatusOK {
+			fmt.Println("Loading source:", err.Error())
+			return nil, 0
+		}
+		data, err := ioutil.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		if err != nil {
+			fmt.Println("Loading source:", err.Error())
+			return nil, 0
+		}
+		lines = bytes.Split(data, []byte{'\n'})
+	}
+
+	return getLineFromLines(lines, line, context)
+}
+
+func getLineFromLines(lines [][]byte, line, context int) ([][]byte, int) {
+	if lines == nil {
+		// cached error from ReadFile: return no lines
+		return nil, 0
+	}
+
+	line-- // stack trace lines are 1-indexed
+	start := line - context
+	var idx int
+	if start < 0 {
+		start = 0
+		idx = line
+	} else {
+		idx = context
+	}
+	end := line + context + 1
+	if line >= len(lines) {
+		return nil, 0
+	}
+	if end > len(lines) {
+		end = len(lines)
+	}
+	return lines[start:end], idx
+}

cmd/stcrashreceiver/stcrashreceiver.go

@@ -0,0 +1,160 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Command stcrashreceiver is a trivial HTTP server that allows two things:
+//
+// - uploading files (crash reports) named like a SHA256 hash using a PUT request
+// - checking whether such file exists using a HEAD request
+//
+// Typically this should be deployed behind something that manages HTTPS.
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"flag"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+const maxRequestSize = 1 << 20 // 1 MiB
+
+func main() {
+	dir := flag.String("dir", ".", "Directory to store reports in")
+	dsn := flag.String("dsn", "", "Sentry DSN")
+	listen := flag.String("listen", ":22039", "HTTP listen address")
+	flag.Parse()
+
+	cr := &crashReceiver{
+		dir: *dir,
+		dsn: *dsn,
+	}
+
+	log.SetOutput(os.Stdout)
+	if err := http.ListenAndServe(*listen, cr); err != nil {
+		log.Fatalln("HTTP serve:", err)
+	}
+}
+
+type crashReceiver struct {
+	dir string
+	dsn string
+}
+
+func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	// The final path component should be a SHA256 hash in hex, so 64 hex
+	// characters. We don't care about case on the request but use lower
+	// case internally.
+	reportID := strings.ToLower(path.Base(req.URL.Path))
+	if len(reportID) != 64 {
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+	for _, c := range reportID {
+		if c >= 'a' && c <= 'f' {
+			continue
+		}
+		if c >= '0' && c <= '9' {
+			continue
+		}
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+
+	// The location of the report on disk, compressed
+	fullPath := filepath.Join(r.dir, r.dirFor(reportID), reportID) + ".gz"
+
+	switch req.Method {
+	case http.MethodGet:
+		r.serveGet(fullPath, w, req)
+	case http.MethodHead:
+		r.serveHead(fullPath, w, req)
+	case http.MethodPut:
+		r.servePut(reportID, fullPath, w, req)
+	default:
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+// serveGet responds to GET requests by serving the uncompressed report.
+func (r *crashReceiver) serveGet(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	fd, err := os.Open(fullPath)
+	if err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+		return
+	}
+
+	defer fd.Close()
+	gr, err := gzip.NewReader(fd)
+	if err != nil {
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+	_, _ = io.Copy(w, gr) // best effort
+}
+
+// serveHead responds to HEAD requests by checking if the named report
+// already exists in the system.
+func (r *crashReceiver) serveHead(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	if _, err := os.Lstat(fullPath); err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+	}
+}
+
+// servePut accepts and stores the given report.
+func (r *crashReceiver) servePut(reportID, fullPath string, w http.ResponseWriter, req *http.Request) {
+	// Ensure the destination directory exists
+	if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
+		log.Println("Creating directory:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Read at most maxRequestSize of report data.
+	log.Println("Receiving report", reportID)
+	lr := io.LimitReader(req.Body, maxRequestSize)
+	bs, err := ioutil.ReadAll(lr)
+	if err != nil {
+		log.Println("Reading report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Compress the report for storage
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	_, _ = gw.Write(bs) // can't fail
+	gw.Close()
+
+	// Create an output file with the compressed report
+	err = ioutil.WriteFile(fullPath, buf.Bytes(), 0644)
+	if err != nil {
+		log.Println("Saving report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Send the report to Sentry
+	if r.dsn != "" {
+		go func() {
+			// There's no need for the client to have to wait for this part.
+			if err := sendReport(r.dsn, reportID, bs); err != nil {
+				log.Println("Failed to send report:", err)
+			}
+		}()
+	}
+}
+
+// 01234567890abcdef... => 01/23
+func (r *crashReceiver) dirFor(base string) string {
+	return filepath.Join(base[0:2], base[2:4])
+}

cmd/stdiscosrv/apisrv.go

@@ -18,6 +18,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -279,6 +280,10 @@ func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addre
 		dbAddrs[i].Expires = expire
 	}
 
+	// The address slice must always be sorted for database merges to work
+	// properly.
+	sort.Sort(databaseAddressOrder(dbAddrs))
+
 	seen := now.UnixNano()
 	if s.repl != nil {
 		s.repl.send(key, dbAddrs, seen)
@@ -295,28 +300,66 @@ func certificateBytes(req *http.Request) []byte {
 		return req.TLS.PeerCertificates[0].Raw
 	}
 
+	var bs []byte
+
 	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
-		bs := []byte(hdr)
-		// The certificate is in PEM format but with spaces for newlines. We
-		// need to reinstate the newlines for the PEM decoder. But we need to
-		// leave the spaces in the BEGIN and END lines - the first and last
-		// space - alone.
-		firstSpace := bytes.Index(bs, []byte(" "))
-		lastSpace := bytes.LastIndex(bs, []byte(" "))
-		for i := firstSpace + 1; i < lastSpace; i++ {
-			if bs[i] == ' ' {
-				bs[i] = '\n'
+		if strings.Contains(hdr, "%") {
+			// Nginx using $ssl_client_escaped_cert
+			// The certificate is in PEM format with url encoding.
+			// We need to decode for the PEM decoder
+			hdr, err := url.QueryUnescape(hdr)
+			if err != nil {
+				// Decoding failed
+				return nil
+			}
+
+			bs = []byte(hdr)
+		} else {
+			// Nginx using $ssl_client_cert
+			// The certificate is in PEM format but with spaces for newlines. We
+			// need to reinstate the newlines for the PEM decoder. But we need to
+			// leave the spaces in the BEGIN and END lines - the first and last
+			// space - alone.
+			bs = []byte(hdr)
+			firstSpace := bytes.Index(bs, []byte(" "))
+			lastSpace := bytes.LastIndex(bs, []byte(" "))
+			for i := firstSpace + 1; i < lastSpace; i++ {
+				if bs[i] == ' ' {
+					bs[i] = '\n'
+				}
 			}
 		}
-		block, _ := pem.Decode(bs)
-		if block == nil {
+	} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
+		// Traefik 2 passtlsclientcert
+		// The certificate is in PEM format with url encoding but without newlines
+		// and start/end statements. We need to decode, reinstate the newlines every 64
+		// character and add statements for the PEM decoder
+		hdr, err := url.QueryUnescape(hdr)
+		if err != nil {
 			// Decoding failed
 			return nil
 		}
-		return block.Bytes
+
+		for i := 64; i < len(hdr); i += 65 {
+			hdr = hdr[:i] + "\n" + hdr[i:]
+		}
+
+		hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
+		hdr = hdr + "\n-----END CERTIFICATE-----\n"
+		bs = []byte(hdr)
 	}
 
-	return nil
+	if bs == nil {
+		return nil
+	}
+
+	block, _ := pem.Decode(bs)
+	if block == nil {
+		// Decoding failed
+		return nil
+	}
+
+	return block.Bytes
 }
 
 // fixupAddresses checks the list of addresses, removing invalid ones and
@@ -335,7 +378,21 @@ func fixupAddresses(remote net.IP, addresses []string) []string {
 		}
 
 		ip := net.ParseIP(host)
+
+		// Some classes of IP are no-go.
+		if ip.IsLoopback() || ip.IsMulticast() {
+			continue
+		}
+
 		if host == "" || ip.IsUnspecified() {
+			// Replace the unspecified IP with the request source.
+
+			// ... unless the request source is the loopback address or
+			// multicast/unspecified (can't happen, really).
+			if remote.IsLoopback() || remote.IsMulticast() || remote.IsUnspecified() {
+				continue
+			}
+
 			// Do not use IPv6 remote address if requested scheme is ...4
 			// (i.e., tcp4, etc.)
 			if strings.HasSuffix(uri.Scheme, "4") && remote.To4() == nil {

cmd/stdiscosrv/apisrv_test.go

@@ -0,0 +1,65 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"net"
+	"testing"
+)
+
+func TestFixupAddresses(t *testing.T) {
+	cases := []struct {
+		remote net.IP
+		in     []string
+		out    []string
+	}{
+		{ // verbatim passthrough
+			in:  []string{"tcp://1.2.3.4:22000"},
+			out: []string{"tcp://1.2.3.4:22000"},
+		}, { // unspecified replaced by remote
+			remote: net.ParseIP("1.2.3.4"),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://1.2.3.4:22000", "tcp://192.0.2.42:22000"},
+		}, { // unspecified not used as replacement
+			remote: net.ParseIP("0.0.0.0"),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // unspecified not used as replacement
+			remote: net.ParseIP("::"),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // localhost not used as replacement
+			remote: net.ParseIP("127.0.0.1"),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // localhost not used as replacement
+			remote: net.ParseIP("::1"),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // multicast not used as replacement
+			remote: net.ParseIP("224.0.0.1"),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // multicast not used as replacement
+			remote: net.ParseIP("ff80::42"),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // explicitly announced weirdness is also filtered
+			remote: net.ParseIP("192.0.2.42"),
+			in:     []string{"tcp://:22000", "tcp://127.1.2.3:22000", "tcp://[::1]:22000", "tcp://[ff80::42]:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		},
+	}
+
+	for _, tc := range cases {
+		out := fixupAddresses(tc.remote, tc.in)
+		if fmt.Sprint(out) != fmt.Sprint(tc.out) {
+			t.Errorf("fixupAddresses(%v, %v) => %v, expected %v", tc.remote, tc.in, out, tc.out)
+		}
+	}
+}

cmd/stdiscosrv/database.go

@@ -5,11 +5,12 @@
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
 //go:generate go run ../../script/protofmt.go database.proto
-//go:generate protoc -I ../../../../../ -I ../../vendor/ -I ../../vendor/github.com/gogo/protobuf/protobuf -I . --gogofast_out=. database.proto
+//go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
 
 package main
 
 import (
+	"log"
 	"sort"
 	"time"
 
@@ -162,10 +163,10 @@ func (s *levelDBStore) Serve() {
 	// Start the statistics serve routine. It will exit with us when
 	// statisticsTrigger is closed.
 	statisticsTrigger := make(chan struct{})
-	defer close(statisticsTrigger)
 	statisticsDone := make(chan struct{})
 	go s.statisticsServe(statisticsTrigger, statisticsDone)
 
+loop:
 	for {
 		select {
 		case fn := <-s.inbox:
@@ -184,12 +185,18 @@ func (s *levelDBStore) Serve() {
 
 		case <-s.stop:
 			// We're done.
-			return
+			close(statisticsTrigger)
+			break loop
 		}
 	}
+
+	// Also wait for statisticsServe to return
+	<-statisticsDone
 }
 
 func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
+	defer close(done)
+
 	for range trigger {
 		t0 := time.Now()
 		nowNanos := t0.UnixNano()
@@ -257,12 +264,15 @@ func (s *levelDBStore) Stop() {
 // chosen for any duplicates.
 func merge(a, b DatabaseRecord) DatabaseRecord {
 	// Both lists must be sorted for this to work.
-	sort.Slice(a.Addresses, func(i, j int) bool {
-		return a.Addresses[i].Address < a.Addresses[j].Address
-	})
-	sort.Slice(b.Addresses, func(i, j int) bool {
-		return b.Addresses[i].Address < b.Addresses[j].Address
-	})
+	if !sort.IsSorted(databaseAddressOrder(a.Addresses)) {
+		log.Println("Warning: bug: addresses not correctly sorted in merge")
+		a.Addresses = sortedAddressCopy(a.Addresses)
+	}
+	if !sort.IsSorted(databaseAddressOrder(b.Addresses)) {
+		// no warning because this is the side we read from disk and it may
+		// legitimately predate correct sorting.
+		b.Addresses = sortedAddressCopy(b.Addresses)
+	}
 
 	res := DatabaseRecord{
 		Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
@@ -346,3 +356,24 @@ func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
 	}
 	return addrs
 }
+
+func sortedAddressCopy(addrs []DatabaseAddress) []DatabaseAddress {
+	sorted := make([]DatabaseAddress, len(addrs))
+	copy(sorted, addrs)
+	sort.Sort(databaseAddressOrder(sorted))
+	return sorted
+}
+
+type databaseAddressOrder []DatabaseAddress
+
+func (s databaseAddressOrder) Less(a, b int) bool {
+	return s[a].Address < s[b].Address
+}
+
+func (s databaseAddressOrder) Swap(a, b int) {
+	s[a], s[b] = s[b], s[a]
+}
+
+func (s databaseAddressOrder) Len() int {
+	return len(s)
+}

cmd/stdiscosrv/database.pb.go

@@ -1,25 +1,16 @@
 // Code generated by protoc-gen-gogo. DO NOT EDIT.
 // source: database.proto
 
-/*
-	Package main is a generated protocol buffer package.
-
-	It is generated from these files:
-		database.proto
-
-	It has these top-level messages:
-		DatabaseRecord
-		ReplicationRecord
-		DatabaseAddress
-*/
 package main
 
-import proto "github.com/gogo/protobuf/proto"
-import fmt "fmt"
-import math "math"
-import _ "github.com/gogo/protobuf/gogoproto"
-
-import io "io"
+import (
+	fmt "fmt"
+	_ "github.com/gogo/protobuf/gogoproto"
+	proto "github.com/gogo/protobuf/proto"
+	io "io"
+	math "math"
+	math_bits "math/bits"
+)
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
@@ -30,50 +21,158 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
 
 type DatabaseRecord struct {
-	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses" json:"addresses"`
+	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
 	Misses    int32             `protobuf:"varint,2,opt,name=misses,proto3" json:"misses,omitempty"`
 	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
 	Missed    int64             `protobuf:"varint,4,opt,name=missed,proto3" json:"missed,omitempty"`
 }
 
-func (m *DatabaseRecord) Reset()                    { *m = DatabaseRecord{} }
-func (m *DatabaseRecord) String() string            { return proto.CompactTextString(m) }
-func (*DatabaseRecord) ProtoMessage()               {}
-func (*DatabaseRecord) Descriptor() ([]byte, []int) { return fileDescriptorDatabase, []int{0} }
+func (m *DatabaseRecord) Reset()         { *m = DatabaseRecord{} }
+func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
+func (*DatabaseRecord) ProtoMessage()    {}
+func (*DatabaseRecord) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{0}
+}
+func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseRecord.Merge(m, src)
+}
+func (m *DatabaseRecord) XXX_Size() int {
+	return m.Size()
+}
+func (m *DatabaseRecord) XXX_DiscardUnknown() {
+	xxx_messageInfo_DatabaseRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DatabaseRecord proto.InternalMessageInfo
 
 type ReplicationRecord struct {
 	Key       string            `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
-	Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses" json:"addresses"`
+	Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses"`
 	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
 }
 
-func (m *ReplicationRecord) Reset()                    { *m = ReplicationRecord{} }
-func (m *ReplicationRecord) String() string            { return proto.CompactTextString(m) }
-func (*ReplicationRecord) ProtoMessage()               {}
-func (*ReplicationRecord) Descriptor() ([]byte, []int) { return fileDescriptorDatabase, []int{1} }
+func (m *ReplicationRecord) Reset()         { *m = ReplicationRecord{} }
+func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
+func (*ReplicationRecord) ProtoMessage()    {}
+func (*ReplicationRecord) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{1}
+}
+func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ReplicationRecord.Merge(m, src)
+}
+func (m *ReplicationRecord) XXX_Size() int {
+	return m.Size()
+}
+func (m *ReplicationRecord) XXX_DiscardUnknown() {
+	xxx_messageInfo_ReplicationRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ReplicationRecord proto.InternalMessageInfo
 
 type DatabaseAddress struct {
 	Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
 	Expires int64  `protobuf:"varint,2,opt,name=expires,proto3" json:"expires,omitempty"`
 }
 
-func (m *DatabaseAddress) Reset()                    { *m = DatabaseAddress{} }
-func (m *DatabaseAddress) String() string            { return proto.CompactTextString(m) }
-func (*DatabaseAddress) ProtoMessage()               {}
-func (*DatabaseAddress) Descriptor() ([]byte, []int) { return fileDescriptorDatabase, []int{2} }
+func (m *DatabaseAddress) Reset()         { *m = DatabaseAddress{} }
+func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
+func (*DatabaseAddress) ProtoMessage()    {}
+func (*DatabaseAddress) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{2}
+}
+func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseAddress.Merge(m, src)
+}
+func (m *DatabaseAddress) XXX_Size() int {
+	return m.Size()
+}
+func (m *DatabaseAddress) XXX_DiscardUnknown() {
+	xxx_messageInfo_DatabaseAddress.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DatabaseAddress proto.InternalMessageInfo
 
 func init() {
 	proto.RegisterType((*DatabaseRecord)(nil), "main.DatabaseRecord")
 	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
 	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
 }
+
+func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
+
+var fileDescriptor_b90fe3356ea5df07 = []byte{
+	// 270 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
+	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
+	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
+	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
+	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
+	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
+	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
+	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
+	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
+	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
+	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
+	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
+	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
+	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
+	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
+	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
+	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
+}
+
 func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -81,44 +180,51 @@ func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Addresses) > 0 {
-		for _, msg := range m.Addresses {
-			dAtA[i] = 0xa
-			i++
-			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
-			n, err := msg.MarshalTo(dAtA[i:])
-			if err != nil {
-				return 0, err
-			}
-			i += n
-		}
-	}
-	if m.Misses != 0 {
-		dAtA[i] = 0x10
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+	if m.Missed != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+		i--
+		dAtA[i] = 0x20
 	}
 	if m.Seen != 0 {
-		dAtA[i] = 0x18
-		i++
 		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
 	}
-	if m.Missed != 0 {
-		dAtA[i] = 0x20
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+	if m.Misses != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+		i--
+		dAtA[i] = 0x10
 	}
-	return i, nil
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
 }
 
 func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -126,40 +232,48 @@ func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Key) > 0 {
-		dAtA[i] = 0xa
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
-		i += copy(dAtA[i:], m.Key)
+	if m.Seen != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
 	}
 	if len(m.Addresses) > 0 {
-		for _, msg := range m.Addresses {
-			dAtA[i] = 0x12
-			i++
-			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
-			n, err := msg.MarshalTo(dAtA[i:])
-			if err != nil {
-				return 0, err
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
 			}
-			i += n
+			i--
+			dAtA[i] = 0x12
 		}
 	}
-	if m.Seen != 0 {
-		dAtA[i] = 0x18
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+	if len(m.Key) > 0 {
+		i -= len(m.Key)
+		copy(dAtA[i:], m.Key)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		i--
+		dAtA[i] = 0xa
 	}
-	return i, nil
+	return len(dAtA) - i, nil
 }
 
 func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -167,52 +281,45 @@ func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Address) > 0 {
-		dAtA[i] = 0xa
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
-		i += copy(dAtA[i:], m.Address)
-	}
 	if m.Expires != 0 {
-		dAtA[i] = 0x10
-		i++
 		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
+		i--
+		dAtA[i] = 0x10
 	}
-	return i, nil
+	if len(m.Address) > 0 {
+		i -= len(m.Address)
+		copy(dAtA[i:], m.Address)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
 }
 
-func encodeFixed64Database(dAtA []byte, offset int, v uint64) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	dAtA[offset+4] = uint8(v >> 32)
-	dAtA[offset+5] = uint8(v >> 40)
-	dAtA[offset+6] = uint8(v >> 48)
-	dAtA[offset+7] = uint8(v >> 56)
-	return offset + 8
-}
-func encodeFixed32Database(dAtA []byte, offset int, v uint32) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	return offset + 4
-}
 func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
+	offset -= sovDatabase(v)
+	base := offset
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
 		v >>= 7
 		offset++
 	}
 	dAtA[offset] = uint8(v)
-	return offset + 1
+	return base
 }
 func (m *DatabaseRecord) Size() (n int) {
+	if m == nil {
+		return 0
+	}
 	var l int
 	_ = l
 	if len(m.Addresses) > 0 {
@@ -234,6 +341,9 @@ func (m *DatabaseRecord) Size() (n int) {
 }
 
 func (m *ReplicationRecord) Size() (n int) {
+	if m == nil {
+		return 0
+	}
 	var l int
 	_ = l
 	l = len(m.Key)
@@ -253,6 +363,9 @@ func (m *ReplicationRecord) Size() (n int) {
 }
 
 func (m *DatabaseAddress) Size() (n int) {
+	if m == nil {
+		return 0
+	}
 	var l int
 	_ = l
 	l = len(m.Address)
@@ -266,14 +379,7 @@ func (m *DatabaseAddress) Size() (n int) {
 }
 
 func sovDatabase(x uint64) (n int) {
-	for {
-		n++
-		x >>= 7
-		if x == 0 {
-			break
-		}
-	}
-	return n
+	return (math_bits.Len64(x|1) + 6) / 7
 }
 func sozDatabase(x uint64) (n int) {
 	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
@@ -293,7 +399,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -321,7 +427,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= (int(b) & 0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -330,6 +436,9 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -352,7 +461,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Misses |= (int32(b) & 0x7F) << shift
+				m.Misses |= int32(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -371,7 +480,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= (int64(b) & 0x7F) << shift
+				m.Seen |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -390,7 +499,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Missed |= (int64(b) & 0x7F) << shift
+				m.Missed |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -404,6 +513,9 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -431,7 +543,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -459,7 +571,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -469,6 +581,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -488,7 +603,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= (int(b) & 0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -497,6 +612,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -519,7 +637,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= (int64(b) & 0x7F) << shift
+				m.Seen |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -533,6 +651,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -560,7 +681,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -588,7 +709,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -598,6 +719,9 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -617,7 +741,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Expires |= (int64(b) & 0x7F) << shift
+				m.Expires |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -631,6 +755,9 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -646,6 +773,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 func skipDatabase(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0
+	depth := 0
 	for iNdEx < l {
 		var wire uint64
 		for shift := uint(0); ; shift += 7 {
@@ -677,10 +805,8 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			return iNdEx, nil
 		case 1:
 			iNdEx += 8
-			return iNdEx, nil
 		case 2:
 			var length int
 			for shift := uint(0); ; shift += 7 {
@@ -697,76 +823,34 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			iNdEx += length
 			if length < 0 {
 				return 0, ErrInvalidLengthDatabase
 			}
-			return iNdEx, nil
+			iNdEx += length
 		case 3:
-			for {
-				var innerWire uint64
-				var start int = iNdEx
-				for shift := uint(0); ; shift += 7 {
-					if shift >= 64 {
-						return 0, ErrIntOverflowDatabase
-					}
-					if iNdEx >= l {
-						return 0, io.ErrUnexpectedEOF
-					}
-					b := dAtA[iNdEx]
-					iNdEx++
-					innerWire |= (uint64(b) & 0x7F) << shift
-					if b < 0x80 {
-						break
-					}
-				}
-				innerWireType := int(innerWire & 0x7)
-				if innerWireType == 4 {
-					break
-				}
-				next, err := skipDatabase(dAtA[start:])
-				if err != nil {
-					return 0, err
-				}
-				iNdEx = start + next
-			}
-			return iNdEx, nil
+			depth++
 		case 4:
-			return iNdEx, nil
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupDatabase
+			}
+			depth--
 		case 5:
 			iNdEx += 4
-			return iNdEx, nil
 		default:
 			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
 		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthDatabase
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
 	}
-	panic("unreachable")
+	return 0, io.ErrUnexpectedEOF
 }
 
 var (
-	ErrInvalidLengthDatabase = fmt.Errorf("proto: negative length found during unmarshaling")
-	ErrIntOverflowDatabase   = fmt.Errorf("proto: integer overflow")
+	ErrInvalidLengthDatabase        = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowDatabase          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupDatabase = fmt.Errorf("proto: unexpected end of group")
 )
-
-func init() { proto.RegisterFile("database.proto", fileDescriptorDatabase) }
-
-var fileDescriptorDatabase = []byte{
-	// 264 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4b, 0x49, 0x2c, 0x49,
-	0x4c, 0x4a, 0x2c, 0x4e, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0xc9, 0x4d, 0xcc, 0xcc,
-	0x93, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf, 0x4f,
-	0xcf, 0xd7, 0x07, 0x4b, 0x26, 0x95, 0xa6, 0x81, 0x79, 0x60, 0x0e, 0x98, 0x05, 0xd1, 0xa4, 0xd4,
-	0xcf, 0xc8, 0xc5, 0xe7, 0x02, 0x35, 0x27, 0x28, 0x35, 0x39, 0xbf, 0x28, 0x45, 0xc8, 0x92, 0x8b,
-	0x33, 0x31, 0x25, 0xa5, 0x28, 0xb5, 0xb8, 0x38, 0xb5, 0x58, 0x82, 0x51, 0x81, 0x59, 0x83, 0xdb,
-	0x48, 0x54, 0x0f, 0x64, 0xb6, 0x1e, 0x4c, 0xa1, 0x23, 0x44, 0xda, 0x89, 0xe5, 0xc4, 0x3d, 0x79,
-	0x86, 0x20, 0x84, 0x6a, 0x21, 0x31, 0x2e, 0xb6, 0xdc, 0x4c, 0xb0, 0x3e, 0x26, 0x05, 0x46, 0x0d,
-	0xd6, 0x20, 0x28, 0x4f, 0x48, 0x88, 0x8b, 0xa5, 0x38, 0x35, 0x35, 0x4f, 0x82, 0x59, 0x81, 0x51,
-	0x83, 0x39, 0x08, 0xcc, 0x86, 0xab, 0x4d, 0x91, 0x60, 0x01, 0x8b, 0x42, 0x79, 0x4a, 0x25, 0x5c,
-	0x82, 0x41, 0xa9, 0x05, 0x39, 0x99, 0xc9, 0x89, 0x25, 0x99, 0xf9, 0x79, 0x50, 0x37, 0x09, 0x70,
-	0x31, 0x67, 0xa7, 0x56, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0x70, 0x06, 0x81, 0x98, 0xa8, 0xae, 0x64,
-	0x22, 0xc9, 0x95, 0x58, 0x5c, 0xa3, 0xe4, 0xca, 0xc5, 0x8f, 0xa6, 0x4f, 0x48, 0x82, 0x8b, 0x1d,
-	0xaa, 0x07, 0x6a, 0x2f, 0x8c, 0x0b, 0x92, 0x49, 0xad, 0x28, 0xc8, 0x2c, 0x82, 0xfa, 0x93, 0x39,
-	0x08, 0xc6, 0x75, 0x12, 0x38, 0xf1, 0x50, 0x8e, 0xe1, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x8f, 0xe4,
-	0x18, 0x1f, 0x3c, 0x92, 0x63, 0x4c, 0x62, 0x03, 0x87, 0xb3, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff,
-	0x6a, 0x22, 0xa2, 0x85, 0xae, 0x01, 0x00, 0x00,
-}

cmd/stdiscosrv/database.proto

@@ -8,9 +8,12 @@ syntax = "proto3";
 
 package main;
 
-import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+import "repos/protobuf/gogoproto/gogo.proto";
 
 option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.goproto_unkeyed_all) = false;
+option (gogoproto.goproto_unrecognized_all) = false;
+option (gogoproto.goproto_sizecache_all) = false;
 
 message DatabaseRecord {
     repeated DatabaseAddress addresses = 1 [(gogoproto.nullable) = false];

cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv

@@ -0,0 +1,4 @@
+[stdiscosrv]
+title=Syncthing discovery server
+description=Lets syncthing clients discover each other
+ports=8443/tcp

cmd/stdiscosrv/etc/linux-systemd/default

@@ -0,0 +1,3 @@
+# Default settings for syncthing-relaysrv (strelaysrv).
+## Add Options here:
+DISCOSRV_OPTS=

cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=Syncthing Discovery Server
+After=network.target
+Documentation=man:stdiscosrv(1)
+
+[Service]
+WorkingDirectory=/var/lib/syncthing-discosrv
+EnvironmentFile=/etc/default/syncthing-discosrv
+ExecStart=/usr/bin/stdiscosrv $DISCOSRV_OPTS
+
+# Hardening
+User=syncthing-discosrv
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-discosrv
+NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
+
+[Install]
+WantedBy=multi-user.target
+Alias=syncthing-discosrv.service

cmd/stdiscosrv/main.go

@@ -9,17 +9,15 @@ package main
 import (
 	"crypto/tls"
 	"flag"
-	"fmt"
 	"log"
 	"net"
 	"net/http"
 	"os"
-	"runtime"
-	"strconv"
 	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -65,34 +63,11 @@ var levelDBOptions = &opt.Options{
 	WriteBuffer: 32 << 20, // default 4<<20
 }
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`stdiscosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
-}
-
 var (
 	debug = false
 )
 
 func main() {
-	const (
-		cleanIntv = 1 * time.Hour
-		statsIntv = 5 * time.Minute
-	)
-
 	var listen string
 	var dir string
 	var metricsListen string
@@ -114,14 +89,18 @@ func main() {
 	flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
 	flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
 	flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
-	log.Println(LongVersion)
+	log.Println(build.LongVersion)
+	if *showVersion {
+		return
+	}
 
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 0)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
@@ -164,7 +143,9 @@ func main() {
 	}
 
 	// Root of the service tree.
-	main := suture.NewSimple("main")
+	main := suture.New("main", suture.Spec{
+		PassThroughPanics: true,
+	})
 
 	// Start the database.
 	db, err := newLevelDBStore(dir)

cmd/stdiscosrv/replication.go

@@ -18,6 +18,9 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
+const replicationReadTimeout = time.Minute
+const replicationHeartbeatInterval = time.Second * 30
+
 type replicator interface {
 	send(key string, addrs []DatabaseAddress, seen int64)
 }
@@ -79,10 +82,22 @@ func (s *replicationSender) Serve() {
 		return
 	}
 
+	heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
+	defer heartBeatTicker.Stop()
+
 	// Send records.
 	buf := make([]byte, 1024)
 	for {
 		select {
+		case <-heartBeatTicker.C:
+			if len(s.outbox) > 0 {
+				// No need to send heartbeats if there are events/prevrious
+				// heartbeats to send, they will keep the connection alive.
+				continue
+			}
+			// Empty replication message is the heartbeat:
+			s.outbox <- ReplicationRecord{}
+
 		case rec := <-s.outbox:
 			// Buffer must hold record plus four bytes for size
 			size := rec.Size()
@@ -106,6 +121,7 @@ func (s *replicationSender) Serve() {
 			if _, err := conn.Write(buf[:4+n]); err != nil {
 				replicationSendsTotal.WithLabelValues("error").Inc()
 				log.Println("Replication write:", err)
+				// Yes, we are loosing the replication event here.
 				return
 			}
 			replicationSendsTotal.WithLabelValues("success").Inc()
@@ -242,7 +258,7 @@ func (l *replicationListener) handle(conn net.Conn) {
 		default:
 		}
 
-		conn.SetReadDeadline(time.Now().Add(time.Minute))
+		conn.SetReadDeadline(time.Now().Add(replicationReadTimeout))
 
 		// First four bytes are the size
 		if _, err := io.ReadFull(conn, buf[:4]); err != nil {
@@ -256,6 +272,12 @@ func (l *replicationListener) handle(conn net.Conn) {
 		if len(buf) < size {
 			buf = make([]byte, size)
 		}
+
+		if size == 0 {
+			// Heartbeat, ignore
+			continue
+		}
+
 		if _, err := io.ReadFull(conn, buf[:size]); err != nil {
 			log.Println("Replication read record:", err)
 			replicationRecvsTotal.WithLabelValues("error").Inc()

cmd/stdiscosrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-discosrv --ingroup syncthing syncthing-discosrv

cmd/stdiscosrv/stats.go

@@ -109,5 +109,15 @@ func init() {
 		databaseKeys, databaseStatisticsSeconds,
 		databaseOperations, databaseOperationSeconds)
 
-	prometheus.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "syncthing_discovery"))
+	processCollectorOpts := prometheus.ProcessCollectorOpts{
+		Namespace: "syncthing_discovery",
+		PidFn: func() (int, error) {
+			return os.Getpid(), nil
+		},
+	}
+
+	prometheus.MustRegister(
+		prometheus.NewProcessCollector(processCollectorOpts),
+	)
+
 }

cmd/stfinddevice/main.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -82,7 +83,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 }
 
 func checkServer(deviceID protocol.DeviceID, server string) checkResult {
-	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger)
 	if err != nil {
 		return checkResult{error: err}
 	}

cmd/stgenfiles/main.go

@@ -82,7 +82,7 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
 		return err
 	}
 
-	_ = os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
+	os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
 
 	t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
 	return os.Chtimes(p1, t, t)

cmd/stindex/dump.go

@@ -16,7 +16,7 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func dump(ldb *db.Instance) {
+func dump(ldb *db.Lowlevel) {
 	it := ldb.NewIterator(nil, nil)
 	for it.Next() {
 		key := it.Key()

cmd/stindex/dumpsize.go

@@ -37,7 +37,7 @@ func (h *ElementHeap) Pop() interface{} {
 	return x
 }
 
-func dumpsize(ldb *db.Instance) {
+func dumpsize(ldb *db.Lowlevel) {
 	h := &ElementHeap{}
 	heap.Init(h)
 

cmd/stindex/idxck.go

@@ -0,0 +1,242 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+type fileInfoKey struct {
+	folder uint32
+	device uint32
+	name   string
+}
+
+type globalKey struct {
+	folder uint32
+	name   string
+}
+
+type sequenceKey struct {
+	folder   uint32
+	sequence uint64
+}
+
+func idxck(ldb *db.Lowlevel) (success bool) {
+	folders := make(map[uint32]string)
+	devices := make(map[uint32]string)
+	deviceToIDs := make(map[string]uint32)
+	fileInfos := make(map[fileInfoKey]protocol.FileInfo)
+	globals := make(map[globalKey]db.VersionList)
+	sequences := make(map[sequenceKey]string)
+	needs := make(map[globalKey]struct{})
+	var localDeviceKey uint32
+	success = true
+
+	it := ldb.NewIterator(nil, nil)
+	for it.Next() {
+		key := it.Key()
+		switch key[0] {
+		case db.KeyTypeDevice:
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+
+			var f protocol.FileInfo
+			err := f.Unmarshal(it.Value())
+			if err != nil {
+				fmt.Println("Unable to unmarshal FileInfo:", err)
+				success = false
+				continue
+			}
+
+			fileInfos[fileInfoKey{folder, device, name}] = f
+
+		case db.KeyTypeGlobal:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			var flv db.VersionList
+			if err := flv.Unmarshal(it.Value()); err != nil {
+				fmt.Println("Unable to unmarshal VersionList:", err)
+				success = false
+				continue
+			}
+			globals[globalKey{folder, name}] = flv
+
+		case db.KeyTypeFolderIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			folders[key] = string(it.Value())
+
+		case db.KeyTypeDeviceIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			devices[key] = string(it.Value())
+			deviceToIDs[string(it.Value())] = key
+			if bytes.Equal(it.Value(), protocol.LocalDeviceID[:]) {
+				localDeviceKey = key
+			}
+
+		case db.KeyTypeSequence:
+			folder := binary.BigEndian.Uint32(key[1:])
+			seq := binary.BigEndian.Uint64(key[5:])
+			val := it.Value()
+			sequences[sequenceKey{folder, seq}] = string(val[9:])
+
+		case db.KeyTypeNeed:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			needs[globalKey{folder, name}] = struct{}{}
+		}
+	}
+
+	if localDeviceKey == 0 {
+		fmt.Println("Missing key for local device in device index (bailing out)")
+		success = false
+		return
+	}
+
+	for fk, fi := range fileInfos {
+		if fk.name != fi.Name {
+			fmt.Printf("Mismatching FileInfo name, %q (key) != %q (actual)\n", fk.name, fi.Name)
+			success = false
+		}
+
+		folder := folders[fk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for FileInfo %q\n", fk.folder, fk.name)
+			success = false
+			continue
+		}
+		if devices[fk.device] == "" {
+			fmt.Printf("Unknown device ID %d for FileInfo %q, folder %q\n", fk.folder, fk.name, folder)
+			success = false
+		}
+
+		if fk.device == localDeviceKey {
+			name, ok := sequences[sequenceKey{fk.folder, uint64(fi.Sequence)}]
+			if !ok {
+				fmt.Printf("Sequence entry missing for FileInfo %q, folder %q, seq %d\n", fi.Name, folder, fi.Sequence)
+				success = false
+				continue
+			}
+			if name != fi.Name {
+				fmt.Printf("Sequence entry refers to wrong name, %q (seq) != %q (FileInfo), folder %q, seq %d\n", name, fi.Name, folder, fi.Sequence)
+				success = false
+			}
+		}
+	}
+
+	for gk, vl := range globals {
+		folder := folders[gk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
+			success = false
+		}
+		for i, fv := range vl.Versions {
+			dev, ok := deviceToIDs[string(fv.Device)]
+			if !ok {
+				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, fv.Device)
+				success = false
+			}
+			fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
+			if !ok {
+				fmt.Printf("VersionList %q, folder %q, entry %d refers to unknown FileInfo\n", gk.name, folder, i)
+				success = false
+			}
+			if !fi.Version.Equal(fv.Version) {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Version, fi.Version)
+				success = false
+			}
+			if fi.IsInvalid() != fv.Invalid {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Invalid, fi.IsInvalid())
+				success = false
+			}
+		}
+
+		// If we need this file we should have a need entry for it. False
+		// positives from needsLocally for deleted files, where we might
+		// legitimately lack an entry if we never had it, and ignored files.
+		if needsLocally(vl) {
+			_, ok := needs[gk]
+			if !ok {
+				dev := deviceToIDs[string(vl.Versions[0].Device)]
+				fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
+				if !fi.IsDeleted() && !fi.IsIgnored() {
+					fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
+				}
+			}
+		}
+	}
+
+	seenSeq := make(map[fileInfoKey]uint64)
+	for sk, name := range sequences {
+		folder := folders[sk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for sequence entry %d, %q\n", sk.folder, sk.sequence, name)
+			success = false
+			continue
+		}
+
+		if prev, ok := seenSeq[fileInfoKey{folder: sk.folder, name: name}]; ok {
+			fmt.Printf("Duplicate sequence entry for %q, folder %q, seq %d (prev %d)\n", name, folder, sk.sequence, prev)
+			success = false
+		}
+		seenSeq[fileInfoKey{folder: sk.folder, name: name}] = sk.sequence
+
+		fi, ok := fileInfos[fileInfoKey{sk.folder, localDeviceKey, name}]
+		if !ok {
+			fmt.Printf("Missing FileInfo for sequence entry %d, folder %q, %q\n", sk.sequence, folder, name)
+			success = false
+			continue
+		}
+		if fi.Sequence != int64(sk.sequence) {
+			fmt.Printf("Sequence mismatch for %q, folder %q, %d (key) != %d (FileInfo)\n", name, folder, sk.sequence, fi.Sequence)
+			success = false
+		}
+	}
+
+	for nk := range needs {
+		folder := folders[nk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for need entry %q\n", nk.folder, nk.name)
+			success = false
+			continue
+		}
+
+		vl, ok := globals[nk]
+		if !ok {
+			fmt.Printf("Missing global for need entry %q, folder %q\n", nk.name, folder)
+			success = false
+			continue
+		}
+
+		if !needsLocally(vl) {
+			fmt.Printf("Need entry for file we don't need, %q, folder %q\n", nk.name, folder)
+			success = false
+		}
+	}
+
+	return
+}
+
+func needsLocally(vl db.VersionList) bool {
+	var lv *protocol.Vector
+	for _, fv := range vl.Versions {
+		if bytes.Equal(fv.Device, protocol.LocalDeviceID[:]) {
+			lv = &fv.Version
+			break
+		}
+	}
+	if lv == nil {
+		return true // proviosinally, it looks like we need the file
+	}
+	return !lv.GreaterEqual(vl.Versions[0].Version)
+}

cmd/stindex/main.go

@@ -21,7 +21,7 @@ func main() {
 	log.SetFlags(0)
 	log.SetOutput(os.Stdout)
 
-	flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize")
+	flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize, idxck")
 
 	flag.Parse()
 
@@ -30,9 +30,7 @@ func main() {
 		path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
 	}
 
-	fmt.Println("Path:", path)
-
-	ldb, err := db.Open(path)
+	ldb, err := db.OpenRO(path)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -41,6 +39,10 @@ func main() {
 		dump(ldb)
 	} else if mode == "dumpsize" {
 		dumpsize(ldb)
+	} else if mode == "idxck" {
+		if !idxck(ldb) {
+			os.Exit(1)
+		}
 	} else {
 		fmt.Println("Unknown mode")
 	}

cmd/strelaypoolsrv/gui/index.html

@@ -187,15 +187,15 @@
     <script type="text/javascript" src="//code.jquery.com/jquery-2.1.4.min.js"></script>
     <script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
     <script type="text/javascript" src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
-    <script type="text/javascript" src="//maps.googleapis.com/maps/api/js"></script>
+    <script type="text/javascript" src="//maps.googleapis.com/maps/api/js?key=AIzaSyDk5WJ8s7ueLKb99X5DbQ-vkWtPDAKqYs0"></script>
   </body>
 
   <script>
   angular.module('syncthing', [
   ])
-  .config(function($httpProvider) {
+  .config(['$httpProvider', function($httpProvider) {
     $httpProvider.defaults.timeout = 5000;
-  })
+  }])
   .filter('bytes', function() {
     return function(bytes, precision) {
       if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) return '-';

cmd/strelaypoolsrv/main.go

@@ -13,7 +13,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
-	"math/rand"
 	"mime"
 	"net"
 	"net/http"
@@ -29,6 +28,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
+	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/relay/client"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -370,10 +370,7 @@ func handleGetRequest(w http.ResponseWriter, r *http.Request) {
 	mut.RUnlock()
 
 	// Shuffle
-	for i := range relays {
-		j := rand.Intn(i + 1)
-		relays[i], relays[j] = relays[j], relays[i]
-	}
+	rand.Shuffle(relays)
 
 	json.NewEncoder(w).Encode(map[string][]*relay{
 		"relays": relays,
@@ -636,7 +633,7 @@ func createTestCertificate() tls.Certificate {
 	}
 
 	certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
-	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 3072)
+	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 20*365)
 	if err != nil {
 		log.Fatalln("Failed to create test X509 key pair:", err)
 	}

cmd/strelaypoolsrv/stats.go

@@ -14,7 +14,16 @@ import (
 )
 
 func init() {
-	prometheus.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "syncthing_relaypoolsrv"))
+	processCollectorOpts := prometheus.ProcessCollectorOpts{
+		Namespace: "syncthing_relaypoolsrv",
+		PidFn: func() (int, error) {
+			return os.Getpid(), nil
+		},
+	}
+
+	prometheus.MustRegister(
+		prometheus.NewProcessCollector(processCollectorOpts),
+	)
 }
 
 var (
@@ -117,7 +126,7 @@ func refreshStats() {
 		go func(rel *relay) {
 			t0 := time.Now()
 			stats := fetchStats(rel)
-			duration := time.Now().Sub(t0).Seconds()
+			duration := time.Since(t0).Seconds()
 			result := "success"
 			if stats == nil {
 				result = "failed"

cmd/strelaysrv/etc/firewall-ufw/strelaysrv

@@ -0,0 +1,9 @@
+[strelaysrv]
+title=Syncthing relay server
+description=Proxies traffic of syncthing client behind firewalls
+ports=22067/tcp
+
+[strelaysrv-metrics]
+title=Syncthing relay metrics
+description=Provides metrics about the syncthing relay server
+ports=22070/tcp

cmd/strelaysrv/etc/linux-systemd/default

@@ -0,0 +1,5 @@
+# Default settings for syncthing-relaysrv (strelaysrv).
+NAT=true
+
+## Add Options here:
+RELAYSRV_OPTS=

cmd/strelaysrv/etc/linux-systemd/strelaysrv.service

@@ -1,17 +1,25 @@
 [Unit]
-Description=Syncthing relay server
+Description=Syncthing Relay Server
 After=network.target
+Documentation=man:strelaysrv(1)
 
 [Service]
-User=strelaysrv
-Group=strelaysrv
-ExecStart=/usr/bin/strelaysrv
-WorkingDirectory=/var/lib/strelaysrv
+WorkingDirectory=/var/lib/syncthing-relaysrv
+EnvironmentFile=/etc/default/syncthing-relaysrv
+ExecStart=/usr/bin/strelaysrv -nat=${NAT} $RELAYSRV_OPTS
 
-PrivateTmp=true
-ProtectSystem=full
-ProtectHome=true
+# Hardening
+User=syncthing-relaysrv
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-relaysrv
 NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
 
 [Install]
 WantedBy=multi-user.target
+Alias=syncthing-relaysrv.service

cmd/strelaysrv/main.go

@@ -14,12 +14,13 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
 	"sync/atomic"
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/relay/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -33,24 +34,6 @@ import (
 	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
 )
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`strelaysrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
-}
-
 var (
 	listen string
 	debug  bool
@@ -116,8 +99,14 @@ func main() {
 	flag.IntVar(&natTimeout, "nat-timeout", 10, "NAT discovery timeout in seconds")
 	flag.BoolVar(&pprofEnabled, "pprof", false, "Enable the built in profiling on the status server")
 	flag.IntVar(&networkBufferSize, "network-buffer", 2048, "Network buffer size (two of these per proxied connection)")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
+	if *showVersion {
+		fmt.Println(build.LongVersion)
+		return
+	}
+
 	if extAddress == "" {
 		extAddress = listen
 	}
@@ -146,7 +135,7 @@ func main() {
 		}
 	}
 
-	log.Println(LongVersion)
+	log.Println(build.LongVersion)
 
 	maxDescriptors, err := osutil.MaximizeOpenFileLimit()
 	if maxDescriptors > 0 {
@@ -166,7 +155,7 @@ func main() {
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 3072)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
@@ -194,7 +183,7 @@ func main() {
 		log.Println("ID:", id)
 	}
 
-	wrapper := config.Wrap("config", config.New(id))
+	wrapper := config.Wrap("config", config.New(id), events.NoopLogger)
 	wrapper.SetOptions(config.OptionsConfiguration{
 		NATLeaseM:   natLease,
 		NATRenewalM: natRenewal,

cmd/strelaysrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-relaysrv --ingroup syncthing syncthing-relaysrv

cmd/strelaysrv/session.go

@@ -22,7 +22,7 @@ import (
 var (
 	sessionMut      = sync.RWMutex{}
 	activeSessions  = make([]*session, 0)
-	pendingSessions = make(map[string]*session, 0)
+	pendingSessions = make(map[string]*session)
 	numProxies      int64
 	bytesProxied    int64
 )

cmd/strelaysrv/status.go

@@ -10,6 +10,8 @@ import (
 	"runtime"
 	"sync/atomic"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/build"
 )
 
 var rc *rateCalculator
@@ -40,10 +42,10 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 
 	sessionMut.Lock()
 	// This can potentially be double the number of pending sessions, as each session has two keys, one for each side.
-	status["version"] = Version
-	status["buildHost"] = BuildHost
-	status["buildUser"] = BuildUser
-	status["buildDate"] = BuildDate
+	status["version"] = build.Version
+	status["buildHost"] = build.Host
+	status["buildUser"] = build.User
+	status["buildDate"] = build.Date
 	status["startTime"] = rc.startTime
 	status["uptimeSeconds"] = time.Since(rc.startTime) / time.Second
 	status["numPendingSessionKeys"] = len(pendingSessions)
@@ -86,9 +88,9 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 }
 
 type rateCalculator struct {
+	counter   *int64 // atomic, must remain 64-bit aligned
 	rates     []int64
 	prev      int64
-	counter   *int64
 	startTime time.Time
 }
 

cmd/stupgrades/main.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"os"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/upgrade"
+)
+
+const defaultURL = "https://api.github.com/repos/syncthing/syncthing/releases?per_page=25"
+
+func main() {
+	url := flag.String("u", defaultURL, "GitHub releases url")
+	flag.Parse()
+
+	rels := upgrade.FetchLatestReleases(*url, "")
+	if rels == nil {
+		// An error was already logged
+		os.Exit(1)
+	}
+
+	sort.Sort(upgrade.SortByRelease(rels))
+	rels = filterForLatest(rels)
+
+	if err := json.NewEncoder(os.Stdout).Encode(rels); err != nil {
+		os.Exit(1)
+	}
+}
+
+// filterForLatest returns the latest stable and prerelease only. If the
+// stable version is newer (comes first in the list) there is no need to go
+// looking for a prerelease at all.
+func filterForLatest(rels []upgrade.Release) []upgrade.Release {
+	var filtered []upgrade.Release
+	var havePre bool
+	for _, rel := range rels {
+		if !rel.Prerelease {
+			// We found a stable version, we're good now.
+			filtered = append(filtered, rel)
+			break
+		}
+		if rel.Prerelease && !havePre {
+			// We remember the first prerelease we find.
+			filtered = append(filtered, rel)
+			havePre = true
+		}
+	}
+	return filtered
+}

cmd/syncthing/auditservice.go

@@ -1,69 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"encoding/json"
-	"io"
-
-	"github.com/syncthing/syncthing/lib/events"
-)
-
-// The auditService subscribes to events and writes these in JSON format, one
-// event per line, to the specified writer.
-type auditService struct {
-	w       io.Writer     // audit destination
-	stop    chan struct{} // signals time to stop
-	started chan struct{} // signals startup complete
-	stopped chan struct{} // signals stop complete
-}
-
-func newAuditService(w io.Writer) *auditService {
-	return &auditService{
-		w:       w,
-		stop:    make(chan struct{}),
-		started: make(chan struct{}),
-		stopped: make(chan struct{}),
-	}
-}
-
-// Serve runs the audit service.
-func (s *auditService) Serve() {
-	defer close(s.stopped)
-	sub := events.Default.Subscribe(events.AllEvents)
-	defer events.Default.Unsubscribe(sub)
-	enc := json.NewEncoder(s.w)
-
-	// We're ready to start processing events.
-	close(s.started)
-
-	for {
-		select {
-		case ev := <-sub.C():
-			enc.Encode(ev)
-		case <-s.stop:
-			return
-		}
-	}
-}
-
-// Stop stops the audit service.
-func (s *auditService) Stop() {
-	close(s.stop)
-}
-
-// WaitForStart returns once the audit service is ready to receive events, or
-// immediately if it's already running.
-func (s *auditService) WaitForStart() {
-	<-s.started
-}
-
-// WaitForStop returns once the audit service has stopped.
-// (Needed by the tests.)
-func (s *auditService) WaitForStop() {
-	<-s.stopped
-}

cmd/syncthing/blockprof.go

@@ -22,11 +22,15 @@ func init() {
 			panic("Couldn't find block profiler")
 		}
 		l.Debugln("Starting block profiling")
-		go saveBlockingProfiles(profiler)
+		go func() {
+			err := saveBlockingProfiles(profiler) // Only returns on error
+			l.Warnln("Block profiler failed:", err)
+			panic("Block profiler failed")
+		}()
 	}
 }
 
-func saveBlockingProfiles(profiler *pprof.Profile) {
+func saveBlockingProfiles(profiler *pprof.Profile) error {
 	runtime.SetBlockProfileRate(1)
 
 	t0 := time.Now()
@@ -35,16 +39,16 @@ func saveBlockingProfiles(profiler *pprof.Profile) {
 
 		fd, err := os.Create(fmt.Sprintf("block-%05d-%07d.pprof", syscall.Getpid(), startms))
 		if err != nil {
-			panic(err)
+			return err
 		}
 		err = profiler.WriteTo(fd, 0)
 		if err != nil {
-			panic(err)
+			return err
 		}
 		err = fd.Close()
 		if err != nil {
-			panic(err)
+			return err
 		}
-
 	}
+	return nil
 }

cmd/syncthing/crash_reporting.go

@@ -0,0 +1,152 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/sha256"
+)
+
+const (
+	headRequestTimeout = 10 * time.Second
+	putRequestTimeout  = time.Minute
+)
+
+// uploadPanicLogs attempts to upload all the panic logs in the named
+// directory to the crash reporting server as urlBase. Uploads are attempted
+// with the newest log first.
+//
+// This can can block for a long time. The context can set a final deadline
+// for this.
+func uploadPanicLogs(ctx context.Context, urlBase, dir string) {
+	files, err := filepath.Glob(filepath.Join(dir, "panic-*.log"))
+	if err != nil {
+		l.Warnln("Failed to list panic logs:", err)
+		return
+	}
+
+	sort.Sort(sort.Reverse(sort.StringSlice(files)))
+	for _, file := range files {
+		if strings.Contains(file, ".reported.") {
+			// We've already sent this file. It'll be cleaned out at some
+			// point.
+			continue
+		}
+
+		if err := uploadPanicLog(ctx, urlBase, file); err != nil {
+			l.Warnln("Reporting crash:", err)
+		} else {
+			// Rename the log so we don't have to try to report it again. This
+			// succeeds, or it does not. There is no point complaining about it.
+			_ = os.Rename(file, strings.Replace(file, ".log", ".reported.log", 1))
+		}
+	}
+}
+
+// uploadPanicLog attempts to upload the named panic log to the crash
+// reporting server at urlBase. The panic ID is constructed as the sha256 of
+// the log contents. A HEAD request is made to see if the log has already
+// been reported. If not, a PUT is made with the log contents.
+func uploadPanicLog(ctx context.Context, urlBase, file string) error {
+	data, err := ioutil.ReadFile(file)
+	if err != nil {
+		return err
+	}
+
+	// Remove log lines, for privacy.
+	data = filterLogLines(data)
+
+	hash := fmt.Sprintf("%x", sha256.Sum256(data))
+	l.Infof("Reporting crash found in %s (report ID %s) ...\n", filepath.Base(file), hash[:8])
+
+	url := fmt.Sprintf("%s/%s", urlBase, hash)
+	headReq, err := http.NewRequest(http.MethodHead, url, nil)
+	if err != nil {
+		return err
+	}
+
+	// Set a reasonable timeout on the HEAD request
+	headCtx, headCancel := context.WithTimeout(ctx, headRequestTimeout)
+	defer headCancel()
+	headReq = headReq.WithContext(headCtx)
+
+	resp, err := http.DefaultClient.Do(headReq)
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	if resp.StatusCode == http.StatusOK {
+		// It's known, we're done
+		return nil
+	}
+
+	putReq, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+
+	// Set a reasonable timeout on the PUT request
+	putCtx, putCancel := context.WithTimeout(ctx, putRequestTimeout)
+	defer putCancel()
+	putReq = putReq.WithContext(putCtx)
+
+	resp, err = http.DefaultClient.Do(putReq)
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("upload: %s", resp.Status)
+	}
+
+	return nil
+}
+
+// filterLogLines returns the data without any log lines between the first
+// line and the panic trace. This is done in-place: the original data slice
+// is destroyed.
+func filterLogLines(data []byte) []byte {
+	filtered := data[:0]
+	matched := false
+	for _, line := range bytes.Split(data, []byte("\n")) {
+		switch {
+		case !matched && bytes.HasPrefix(line, []byte("Panic ")):
+			// This begins the panic trace, set the matched flag and append.
+			matched = true
+			fallthrough
+		case len(filtered) == 0 || matched:
+			// This is the first line or inside the panic trace.
+			if len(filtered) > 0 {
+				// We add the newline before rather than after because
+				// bytes.Split sees the \n as *separator* and not line
+				// ender, so ir will generate a last empty line that we
+				// don't really want. (We want to keep blank lines in the
+				// middle of the trace though.)
+				filtered = append(filtered, '\n')
+			}
+			// Remove the device ID prefix. The "plus two" stuff is because
+			// the line will look like "[foo] whatever" and the end variable
+			// will end up pointing at the ] and we want to step over that
+			// and the following space.
+			if end := bytes.Index(line, []byte("]")); end > 1 && end < len(line)-2 && bytes.HasPrefix(line, []byte("[")) {
+				line = line[end+2:]
+			}
+			filtered = append(filtered, line...)
+		}
+	}
+	return filtered
+}

cmd/syncthing/crash_reporting_test.go

@@ -0,0 +1,37 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestFilterLogLines(t *testing.T) {
+	in := []byte(`[ABCD123] syncthing version whatever
+here is more log data
+and more
+...
+and some more
+yet more
+Panic detected at like right now
+here is panic data
+and yet more panic stuff
+`)
+
+	filtered := []byte(`syncthing version whatever
+Panic detected at like right now
+here is panic data
+and yet more panic stuff
+`)
+
+	result := filterLogLines(in)
+	if !bytes.Equal(result, filtered) {
+		t.Logf("%q\n", result)
+		t.Error("it should have been filtered")
+	}
+}

cmd/syncthing/debug.go

@@ -7,22 +7,9 @@
 package main
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
-	l     = logger.DefaultLogger.NewFacility("main", "Main package")
-	httpl = logger.DefaultLogger.NewFacility("http", "REST API")
+	l = logger.DefaultLogger.NewFacility("main", "Main package")
 )
-
-func shouldDebugHTTP() bool {
-	return l.ShouldDebug("http")
-}
-
-func init() {
-	l.SetDebug("main", strings.Contains(os.Getenv("STTRACE"), "main") || os.Getenv("STTRACE") == "all")
-	l.SetDebug("http", strings.Contains(os.Getenv("STTRACE"), "http") || os.Getenv("STTRACE") == "all")
-}

cmd/syncthing/gui_auth.go

@@ -1,141 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"encoding/base64"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/events"
-	"github.com/syncthing/syncthing/lib/rand"
-	"github.com/syncthing/syncthing/lib/sync"
-	"golang.org/x/crypto/bcrypt"
-)
-
-var (
-	sessions    = make(map[string]bool)
-	sessionsMut = sync.NewMutex()
-)
-
-func emitLoginAttempt(success bool, username string) {
-	events.Default.Log(events.LoginAttempt, map[string]interface{}{
-		"success":  success,
-		"username": username,
-	})
-}
-
-func basicAuthAndSessionMiddleware(cookieName string, cfg config.GUIConfiguration, next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if cfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		cookie, err := r.Cookie(cookieName)
-		if err == nil && cookie != nil {
-			sessionsMut.Lock()
-			_, ok := sessions[cookie.Value]
-			sessionsMut.Unlock()
-			if ok {
-				next.ServeHTTP(w, r)
-				return
-			}
-		}
-
-		httpl.Debugln("Sessionless HTTP request with authentication; this is expensive.")
-
-		error := func() {
-			time.Sleep(time.Duration(rand.Intn(100)+100) * time.Millisecond)
-			w.Header().Set("WWW-Authenticate", "Basic realm=\"Authorization Required\"")
-			http.Error(w, "Not Authorized", http.StatusUnauthorized)
-		}
-
-		hdr := r.Header.Get("Authorization")
-		if !strings.HasPrefix(hdr, "Basic ") {
-			error()
-			return
-		}
-
-		hdr = hdr[6:]
-		bs, err := base64.StdEncoding.DecodeString(hdr)
-		if err != nil {
-			error()
-			return
-		}
-
-		fields := bytes.SplitN(bs, []byte(":"), 2)
-		if len(fields) != 2 {
-			error()
-			return
-		}
-
-		// Check if the username is correct, assuming it was sent as UTF-8
-		username := string(fields[0])
-		if username == cfg.User {
-			goto usernameOK
-		}
-
-		// ... check it again, converting it from assumed ISO-8859-1 to UTF-8
-		username = string(iso88591ToUTF8(fields[0]))
-		if username == cfg.User {
-			goto usernameOK
-		}
-
-		// Neither of the possible interpretations match the configured username
-		emitLoginAttempt(false, username)
-		error()
-		return
-
-	usernameOK:
-		// Check password as given (assumes UTF-8 encoding)
-		password := fields[1]
-		if err := bcrypt.CompareHashAndPassword([]byte(cfg.Password), password); err == nil {
-			goto passwordOK
-		}
-
-		// ... check it again, converting it from assumed ISO-8859-1 to UTF-8
-		password = iso88591ToUTF8(password)
-		if err := bcrypt.CompareHashAndPassword([]byte(cfg.Password), password); err == nil {
-			goto passwordOK
-		}
-
-		// Neither of the attempts to verify the password checked out
-		emitLoginAttempt(false, username)
-		error()
-		return
-
-	passwordOK:
-		sessionid := rand.String(32)
-		sessionsMut.Lock()
-		sessions[sessionid] = true
-		sessionsMut.Unlock()
-		http.SetCookie(w, &http.Cookie{
-			Name:   cookieName,
-			Value:  sessionid,
-			MaxAge: 0,
-		})
-
-		emitLoginAttempt(true, username)
-		next.ServeHTTP(w, r)
-	})
-}
-
-// Convert an ISO-8859-1 encoded byte string to UTF-8. Works by the
-// principle that ISO-8859-1 bytes are equivalent to unicode code points,
-// that a rune slice is a list of code points, and that stringifying a slice
-// of runes generates UTF-8 in Go.
-func iso88591ToUTF8(s []byte) []byte {
-	runes := make([]rune, len(s))
-	for i := range s {
-		runes[i] = rune(s[i])
-	}
-	return []byte(string(runes))
-}

cmd/syncthing/gui_csrf.go

@@ -1,142 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bufio"
-	"fmt"
-	"net/http"
-	"os"
-	"strings"
-
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/osutil"
-	"github.com/syncthing/syncthing/lib/rand"
-	"github.com/syncthing/syncthing/lib/sync"
-)
-
-// csrfTokens is a list of valid tokens. It is sorted so that the most
-// recently used token is first in the list. New tokens are added to the front
-// of the list (as it is the most recently used at that time). The list is
-// pruned to a maximum of maxCsrfTokens, throwing away the least recently used
-// tokens.
-var csrfTokens []string
-var csrfMut = sync.NewMutex()
-
-const maxCsrfTokens = 25
-
-// Check for CSRF token on /rest/ URLs. If a correct one is not given, reject
-// the request with 403. For / and /index.html, set a new CSRF cookie if none
-// is currently set.
-func csrfMiddleware(unique string, prefix string, cfg config.GUIConfiguration, next http.Handler) http.Handler {
-	loadCsrfTokens()
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// Allow requests carrying a valid API key
-		if cfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
-			// Set the access-control-allow-origin header for CORS requests
-			// since a valid API key has been provided
-			w.Header().Add("Access-Control-Allow-Origin", "*")
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		if strings.HasPrefix(r.URL.Path, "/rest/debug") {
-			// Debugging functions are only available when explicitly
-			// enabled, and can be accessed without a CSRF token
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		// Allow requests for anything not under the protected path prefix,
-		// and set a CSRF cookie if there isn't already a valid one.
-		if !strings.HasPrefix(r.URL.Path, prefix) {
-			cookie, err := r.Cookie("CSRF-Token-" + unique)
-			if err != nil || !validCsrfToken(cookie.Value) {
-				httpl.Debugln("new CSRF cookie in response to request for", r.URL)
-				cookie = &http.Cookie{
-					Name:  "CSRF-Token-" + unique,
-					Value: newCsrfToken(),
-				}
-				http.SetCookie(w, cookie)
-			}
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		// Verify the CSRF token
-		token := r.Header.Get("X-CSRF-Token-" + unique)
-		if !validCsrfToken(token) {
-			http.Error(w, "CSRF Error", 403)
-			return
-		}
-
-		next.ServeHTTP(w, r)
-	})
-}
-
-func validCsrfToken(token string) bool {
-	csrfMut.Lock()
-	defer csrfMut.Unlock()
-	for i, t := range csrfTokens {
-		if t == token {
-			if i > 0 {
-				// Move this token to the head of the list. Copy the tokens at
-				// the front one step to the right and then replace the token
-				// at the head.
-				copy(csrfTokens[1:], csrfTokens[:i+1])
-				csrfTokens[0] = token
-			}
-			return true
-		}
-	}
-	return false
-}
-
-func newCsrfToken() string {
-	token := rand.String(32)
-
-	csrfMut.Lock()
-	csrfTokens = append([]string{token}, csrfTokens...)
-	if len(csrfTokens) > maxCsrfTokens {
-		csrfTokens = csrfTokens[:maxCsrfTokens]
-	}
-	defer csrfMut.Unlock()
-
-	saveCsrfTokens()
-
-	return token
-}
-
-func saveCsrfTokens() {
-	// We're ignoring errors in here. It's not super critical and there's
-	// nothing relevant we can do about them anyway...
-
-	name := locations[locCsrfTokens]
-	f, err := osutil.CreateAtomic(name)
-	if err != nil {
-		return
-	}
-
-	for _, t := range csrfTokens {
-		fmt.Fprintln(f, t)
-	}
-
-	f.Close()
-}
-
-func loadCsrfTokens() {
-	f, err := os.Open(locations[locCsrfTokens])
-	if err != nil {
-		return
-	}
-	defer f.Close()
-
-	s := bufio.NewScanner(f)
-	for s.Scan() {
-		csrfTokens = append(csrfTokens, s.Text())
-	}
-}

cmd/syncthing/heapprof.go

@@ -23,11 +23,15 @@ func init() {
 			rate = i
 		}
 		l.Debugln("Starting heap profiling")
-		go saveHeapProfiles(rate)
+		go func() {
+			err := saveHeapProfiles(rate) // Only returns on error
+			l.Warnln("Heap profiler failed:", err)
+			panic("Heap profiler failed")
+		}()
 	}
 }
 
-func saveHeapProfiles(rate int) {
+func saveHeapProfiles(rate int) error {
 	runtime.MemProfileRate = rate
 	var memstats, prevMemstats runtime.MemStats
 
@@ -38,21 +42,21 @@ func saveHeapProfiles(rate int) {
 		if memstats.HeapInuse > prevMemstats.HeapInuse {
 			fd, err := os.Create(name + ".tmp")
 			if err != nil {
-				panic(err)
+				return err
 			}
 			err = pprof.WriteHeapProfile(fd)
 			if err != nil {
-				panic(err)
+				return err
 			}
 			err = fd.Close()
 			if err != nil {
-				panic(err)
+				return err
 			}
 
-			_ = os.Remove(name) // Error deliberately ignored
+			os.Remove(name) // Error deliberately ignored
 			err = os.Rename(name+".tmp", name)
 			if err != nil {
-				panic(err)
+				return err
 			}
 
 			prevMemstats = memstats

cmd/syncthing/locations.go

@@ -1,125 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"os"
-	"path/filepath"
-	"runtime"
-	"strings"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/fs"
-)
-
-type locationEnum string
-
-// Use strings as keys to make printout and serialization of the locations map
-// more meaningful.
-const (
-	locConfigFile    locationEnum = "config"
-	locCertFile                   = "certFile"
-	locKeyFile                    = "keyFile"
-	locHTTPSCertFile              = "httpsCertFile"
-	locHTTPSKeyFile               = "httpsKeyFile"
-	locDatabase                   = "database"
-	locLogFile                    = "logFile"
-	locCsrfTokens                 = "csrfTokens"
-	locPanicLog                   = "panicLog"
-	locAuditLog                   = "auditLog"
-	locGUIAssets                  = "GUIAssets"
-	locDefFolder                  = "defFolder"
-)
-
-// Platform dependent directories
-var baseDirs = map[string]string{
-	"config": defaultConfigDir(), // Overridden by -home flag
-	"home":   homeDir(),          // User's home directory, *not* -home flag
-}
-
-// Use the variables from baseDirs here
-var locations = map[locationEnum]string{
-	locConfigFile:    "${config}/config.xml",
-	locCertFile:      "${config}/cert.pem",
-	locKeyFile:       "${config}/key.pem",
-	locHTTPSCertFile: "${config}/https-cert.pem",
-	locHTTPSKeyFile:  "${config}/https-key.pem",
-	locDatabase:      "${config}/index-v0.14.0.db",
-	locLogFile:       "${config}/syncthing.log", // -logfile on Windows
-	locCsrfTokens:    "${config}/csrftokens.txt",
-	locPanicLog:      "${config}/panic-${timestamp}.log",
-	locAuditLog:      "${config}/audit-${timestamp}.log",
-	locGUIAssets:     "${config}/gui",
-	locDefFolder:     "${home}/Sync",
-}
-
-// expandLocations replaces the variables in the location map with actual
-// directory locations.
-func expandLocations() error {
-	for key, dir := range locations {
-		for varName, value := range baseDirs {
-			dir = strings.Replace(dir, "${"+varName+"}", value, -1)
-		}
-		var err error
-		dir, err = fs.ExpandTilde(dir)
-		if err != nil {
-			return err
-		}
-		locations[key] = dir
-	}
-	return nil
-}
-
-// defaultConfigDir returns the default configuration directory, as figured
-// out by various the environment variables present on each platform, or dies
-// trying.
-func defaultConfigDir() string {
-	switch runtime.GOOS {
-	case "windows":
-		if p := os.Getenv("LocalAppData"); p != "" {
-			return filepath.Join(p, "Syncthing")
-		}
-		return filepath.Join(os.Getenv("AppData"), "Syncthing")
-
-	case "darwin":
-		dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
-		if err != nil {
-			l.Fatalln(err)
-		}
-		return dir
-
-	default:
-		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
-			return filepath.Join(xdgCfg, "syncthing")
-		}
-		dir, err := fs.ExpandTilde("~/.config/syncthing")
-		if err != nil {
-			l.Fatalln(err)
-		}
-		return dir
-	}
-}
-
-// homeDir returns the user's home directory, or dies trying.
-func homeDir() string {
-	home, err := fs.ExpandTilde("~")
-	if err != nil {
-		l.Fatalln(err)
-	}
-	return home
-}
-
-func timestampedLoc(key locationEnum) string {
-	// We take the roundtrip via "${timestamp}" instead of passing the path
-	// directly through time.Format() to avoid issues when the path we are
-	// expanding contains numbers; otherwise for example
-	// /home/user2006/.../panic-20060102-150405.log would get both instances of
-	// 2006 replaced by 2015...
-	tpl := locations[key]
-	now := time.Now().Format("20060102-150405")
-	return strings.Replace(tpl, "${timestamp}", now, -1)
-}

cmd/syncthing/mocked_config_test.go

@@ -1,65 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/util"
-)
-
-type mockedConfig struct {
-	gui config.GUIConfiguration
-}
-
-func (c *mockedConfig) GUI() config.GUIConfiguration {
-	return c.gui
-}
-
-func (c *mockedConfig) ListenAddresses() []string {
-	return nil
-}
-
-func (c *mockedConfig) RawCopy() config.Configuration {
-	cfg := config.Configuration{}
-	util.SetDefaults(&cfg.Options)
-	return cfg
-}
-
-func (c *mockedConfig) Options() config.OptionsConfiguration {
-	return config.OptionsConfiguration{}
-}
-
-func (c *mockedConfig) Replace(cfg config.Configuration) (config.Waiter, error) {
-	return nil, nil
-}
-
-func (c *mockedConfig) Subscribe(cm config.Committer) {}
-
-func (c *mockedConfig) Folders() map[string]config.FolderConfiguration {
-	return nil
-}
-
-func (c *mockedConfig) Devices() map[protocol.DeviceID]config.DeviceConfiguration {
-	return nil
-}
-
-func (c *mockedConfig) SetDevice(config.DeviceConfiguration) (config.Waiter, error) {
-	return nil, nil
-}
-
-func (c *mockedConfig) SetDevices([]config.DeviceConfiguration) (config.Waiter, error) {
-	return nil, nil
-}
-
-func (c *mockedConfig) Save() error {
-	return nil
-}
-
-func (c *mockedConfig) RequiresRestart() bool {
-	return false
-}

cmd/syncthing/mocked_connections_test.go

@@ -1,17 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-type mockedConnections struct{}
-
-func (m *mockedConnections) Status() map[string]interface{} {
-	return nil
-}
-
-func (m *mockedConnections) NATType() string {
-	return ""
-}

cmd/syncthing/monitor.go

@@ -8,6 +8,7 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"io"
 	"os"
 	"os/exec"
@@ -17,8 +18,12 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/syncthing"
 )
 
 var (
@@ -32,6 +37,8 @@ const (
 	loopThreshold         = 60 * time.Second
 	logFileAutoCloseDelay = 5 * time.Second
 	logFileMaxOpenTime    = time.Minute
+	panicUploadMaxWait    = 30 * time.Second
+	panicUploadNoticeWait = 10 * time.Second
 )
 
 func monitorMain(runtimeOptions RuntimeOptions) {
@@ -71,9 +78,11 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 	childEnv := childEnv()
 	first := true
 	for {
+		maybeReportPanics()
+
 		if t := time.Since(restarts[0]); t < loopThreshold {
 			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 
 		copy(restarts[0:], restarts[1:])
@@ -84,18 +93,19 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 
 		stderr, err := cmd.StderrPipe()
 		if err != nil {
-			l.Fatalln("stderr:", err)
+			panic(err)
 		}
 
 		stdout, err := cmd.StdoutPipe()
 		if err != nil {
-			l.Fatalln("stdout:", err)
+			panic(err)
 		}
 
 		l.Infoln("Starting syncthing")
 		err = cmd.Start()
 		if err != nil {
-			l.Fatalln(err)
+			l.Warnln("Error starting the main Syncthing process:", err)
+			panic("Error starting the main Syncthing process")
 		}
 
 		stdoutMut.Lock()
@@ -127,7 +137,7 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 		select {
 		case s := <-stopSign:
 			l.Infof("Signal %d received; exiting", s)
-			cmd.Process.Kill()
+			cmd.Process.Signal(sigTerm)
 			<-exit
 			return
 
@@ -141,17 +151,14 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 				// Successful exit indicates an intentional shutdown
 				return
 			} else if exiterr, ok := err.(*exec.ExitError); ok {
-				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
-					switch status.ExitStatus() {
-					case exitUpgrading:
-						// Restart the monitor process to release the .old
-						// binary as part of the upgrade process.
-						l.Infoln("Restarting monitor...")
-						if err = restartMonitor(args); err != nil {
-							l.Warnln("Restart:", err)
-						}
-						return
+				if exiterr.ExitCode() == syncthing.ExitUpgrade.AsInt() {
+					// Restart the monitor process to release the .old
+					// binary as part of the upgrade process.
+					l.Infoln("Restarting monitor...")
+					if err = restartMonitor(args); err != nil {
+						l.Warnln("Restart:", err)
 					}
+					return
 				}
 			}
 		}
@@ -172,6 +179,13 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
 	br := bufio.NewReader(stderr)
 
 	var panicFd *os.File
+	defer func() {
+		if panicFd != nil {
+			_ = panicFd.Close()
+			maybeReportPanics()
+		}
+	}()
+
 	for {
 		line, err := br.ReadString('\n')
 		if err != nil {
@@ -198,7 +212,7 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
 			}
 
 			if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
-				panicFd, err = os.Create(timestampedLoc(locPanicLog))
+				panicFd, err = os.Create(locations.GetTimestamped(locations.PanicLog))
 				if err != nil {
 					l.Warnln("Create panic log:", err)
 					continue
@@ -418,10 +432,10 @@ func (f *autoclosedFile) closerLoop() {
 func childEnv() []string {
 	var env []string
 	for _, str := range os.Environ() {
-		if strings.HasPrefix("STNORESTART=", str) {
+		if strings.HasPrefix(str, "STNORESTART=") {
 			continue
 		}
-		if strings.HasPrefix("STMONITORED=", str) {
+		if strings.HasPrefix(str, "STMONITORED=") {
 			continue
 		}
 		env = append(env, str)
@@ -429,3 +443,39 @@ func childEnv() []string {
 	env = append(env, "STMONITORED=yes")
 	return env
 }
+
+// maybeReportPanics tries to figure out if crash reporting is on or off,
+// and reports any panics it can find if it's enabled. We spend at most
+// panicUploadMaxWait uploading panics...
+func maybeReportPanics() {
+	// Try to get a config to see if/where panics should be reported.
+	cfg, err := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger)
+	if err != nil {
+		l.Warnln("Couldn't load config; not reporting crash")
+		return
+	}
+
+	// Bail if we're not supposed to report panics.
+	opts := cfg.Options()
+	if !opts.CREnabled {
+		return
+	}
+
+	// Set up a timeout on the whole operation.
+	ctx, cancel := context.WithTimeout(context.Background(), panicUploadMaxWait)
+	defer cancel()
+
+	// Print a notice if the upload takes a long time.
+	go func() {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(panicUploadNoticeWait):
+			l.Warnln("Uploading crash reports is taking a while, please wait...")
+		}
+	}()
+
+	// Report the panics.
+	dir := locations.GetBaseDir(locations.ConfigBaseDir)
+	uploadPanicLogs(ctx, opts.CRURL, dir)
+}

cmd/syncthing/perfstats_unix.go

@@ -38,7 +38,10 @@ func savePerfStats(file string) {
 
 	t0 := time.Now()
 	for t := range time.NewTicker(250 * time.Millisecond).C {
-		syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
+		if err := syscall.Getrusage(syscall.RUSAGE_SELF, &rusage); err != nil {
+			continue
+		}
+
 		curTime := time.Now().UnixNano()
 		timeDiff := curTime - prevTime
 		curUsage := rusage.Utime.Nano() + rusage.Stime.Nano()

cmd/syncthing/summaryservice.go

@@ -1,229 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"time"
-
-	"github.com/syncthing/syncthing/lib/events"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/sync"
-	"github.com/thejerf/suture"
-)
-
-// The folderSummaryService adds summary information events (FolderSummary and
-// FolderCompletion) into the event stream at certain intervals.
-type folderSummaryService struct {
-	*suture.Supervisor
-
-	cfg       configIntf
-	model     modelIntf
-	stop      chan struct{}
-	immediate chan string
-
-	// For keeping track of folders to recalculate for
-	foldersMut sync.Mutex
-	folders    map[string]struct{}
-
-	// For keeping track of when the last event request on the API was
-	lastEventReq    time.Time
-	lastEventReqMut sync.Mutex
-}
-
-func newFolderSummaryService(cfg configIntf, m modelIntf) *folderSummaryService {
-	service := &folderSummaryService{
-		Supervisor:      suture.NewSimple("folderSummaryService"),
-		cfg:             cfg,
-		model:           m,
-		stop:            make(chan struct{}),
-		immediate:       make(chan string),
-		folders:         make(map[string]struct{}),
-		foldersMut:      sync.NewMutex(),
-		lastEventReqMut: sync.NewMutex(),
-	}
-
-	service.Add(serviceFunc(service.listenForUpdates))
-	service.Add(serviceFunc(service.calculateSummaries))
-
-	return service
-}
-
-func (c *folderSummaryService) Stop() {
-	c.Supervisor.Stop()
-	close(c.stop)
-}
-
-// listenForUpdates subscribes to the event bus and makes note of folders that
-// need their data recalculated.
-func (c *folderSummaryService) listenForUpdates() {
-	sub := events.Default.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged | events.RemoteDownloadProgress | events.DeviceConnected | events.FolderWatchStateChanged)
-	defer events.Default.Unsubscribe(sub)
-
-	for {
-		// This loop needs to be fast so we don't miss too many events.
-
-		select {
-		case ev := <-sub.C():
-			if ev.Type == events.DeviceConnected {
-				// When a device connects we schedule a refresh of all
-				// folders shared with that device.
-
-				data := ev.Data.(map[string]string)
-				deviceID, _ := protocol.DeviceIDFromString(data["id"])
-
-				c.foldersMut.Lock()
-			nextFolder:
-				for _, folder := range c.cfg.Folders() {
-					for _, dev := range folder.Devices {
-						if dev.DeviceID == deviceID {
-							c.folders[folder.ID] = struct{}{}
-							continue nextFolder
-						}
-					}
-				}
-				c.foldersMut.Unlock()
-
-				continue
-			}
-
-			// The other events all have a "folder" attribute that they
-			// affect. Whenever the local or remote index is updated for a
-			// given folder we make a note of it.
-
-			data := ev.Data.(map[string]interface{})
-			folder := data["folder"].(string)
-
-			switch ev.Type {
-			case events.StateChanged:
-				if data["to"].(string) == "idle" && data["from"].(string) == "syncing" {
-					// The folder changed to idle from syncing. We should do an
-					// immediate refresh to update the GUI. The send to
-					// c.immediate must be nonblocking so that we can continue
-					// handling events.
-
-					c.foldersMut.Lock()
-					select {
-					case c.immediate <- folder:
-						delete(c.folders, folder)
-					default:
-						c.folders[folder] = struct{}{}
-					}
-					c.foldersMut.Unlock()
-				}
-
-			default:
-				// This folder needs to be refreshed whenever we do the next
-				// refresh.
-
-				c.foldersMut.Lock()
-				c.folders[folder] = struct{}{}
-				c.foldersMut.Unlock()
-			}
-
-		case <-c.stop:
-			return
-		}
-	}
-}
-
-// calculateSummaries periodically recalculates folder summaries and
-// completion percentage, and sends the results on the event bus.
-func (c *folderSummaryService) calculateSummaries() {
-	const pumpInterval = 2 * time.Second
-	pump := time.NewTimer(pumpInterval)
-
-	for {
-		select {
-		case <-pump.C:
-			t0 := time.Now()
-			for _, folder := range c.foldersToHandle() {
-				c.sendSummary(folder)
-			}
-
-			// We don't want to spend all our time calculating summaries. Lets
-			// set an arbitrary limit at not spending more than about 30% of
-			// our time here...
-			wait := 2*time.Since(t0) + pumpInterval
-			pump.Reset(wait)
-
-		case folder := <-c.immediate:
-			c.sendSummary(folder)
-
-		case <-c.stop:
-			return
-		}
-	}
-}
-
-// foldersToHandle returns the list of folders needing a summary update, and
-// clears the list.
-func (c *folderSummaryService) foldersToHandle() []string {
-	// We only recalculate summaries if someone is listening to events
-	// (a request to /rest/events has been made within the last
-	// pingEventInterval).
-
-	c.lastEventReqMut.Lock()
-	last := c.lastEventReq
-	c.lastEventReqMut.Unlock()
-	if time.Since(last) > defaultEventTimeout {
-		return nil
-	}
-
-	c.foldersMut.Lock()
-	res := make([]string, 0, len(c.folders))
-	for folder := range c.folders {
-		res = append(res, folder)
-		delete(c.folders, folder)
-	}
-	c.foldersMut.Unlock()
-	return res
-}
-
-// sendSummary send the summary events for a single folder
-func (c *folderSummaryService) sendSummary(folder string) {
-	// The folder summary contains how many bytes, files etc
-	// are in the folder and how in sync we are.
-	data, err := folderSummary(c.cfg, c.model, folder)
-	if err != nil {
-		return
-	}
-	events.Default.Log(events.FolderSummary, map[string]interface{}{
-		"folder":  folder,
-		"summary": data,
-	})
-
-	for _, devCfg := range c.cfg.Folders()[folder].Devices {
-		if devCfg.DeviceID.Equals(myID) {
-			// We already know about ourselves.
-			continue
-		}
-		if _, ok := c.model.Connection(devCfg.DeviceID); !ok {
-			// We're not interested in disconnected devices.
-			continue
-		}
-
-		// Get completion percentage of this folder for the
-		// remote device.
-		comp := jsonCompletion(c.model.Completion(devCfg.DeviceID, folder))
-		comp["folder"] = folder
-		comp["device"] = devCfg.DeviceID.String()
-		events.Default.Log(events.FolderCompletion, comp)
-	}
-}
-
-func (c *folderSummaryService) gotEventRequest() {
-	c.lastEventReqMut.Lock()
-	c.lastEventReq = time.Now()
-	c.lastEventReqMut.Unlock()
-}
-
-// serviceFunc wraps a function to create a suture.Service without stop
-// functionality.
-type serviceFunc func()
-
-func (f serviceFunc) Serve() { f() }
-func (f serviceFunc) Stop()  {}

cmd/todos/main.go

@@ -1,39 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build ignore
-
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"os"
-)
-
-func main() {
-	buf := make([]byte, 4096)
-	var err error
-	for err == nil {
-		n, err := io.ReadFull(os.Stdin, buf)
-		if n > 0 {
-			buf = buf[:n]
-			repl := bytes.Replace(buf, []byte("\n"), []byte("\r\n"), -1)
-			_, err = os.Stdout.Write(repl)
-			if err != nil {
-				fmt.Println(err)
-				os.Exit(1)
-			}
-		}
-		if err == io.EOF {
-			return
-		}
-		buf = buf[:cap(buf)]
-	}
-	fmt.Println(err)
-	os.Exit(1)
-}

cmd/uraggregate/main.go

@@ -0,0 +1,330 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"database/sql"
+	"log"
+	"os"
+	"time"
+
+	_ "github.com/lib/pq"
+)
+
+var dbConn = getEnvDefault("UR_DB_URL", "postgres://user:password@localhost/ur?sslmode=disable")
+
+func getEnvDefault(key, def string) string {
+	if val := os.Getenv(key); val != "" {
+		return val
+	}
+	return def
+}
+
+func main() {
+	log.SetFlags(log.Ltime | log.Ldate)
+	log.SetOutput(os.Stdout)
+
+	db, err := sql.Open("postgres", dbConn)
+	if err != nil {
+		log.Fatalln("database:", err)
+	}
+	err = setupDB(db)
+	if err != nil {
+		log.Fatalln("database:", err)
+	}
+
+	for {
+		runAggregation(db)
+		// Sleep until one minute past next midnight
+		sleepUntilNext(24*time.Hour, 1*time.Minute)
+	}
+}
+
+func runAggregation(db *sql.DB) {
+	since := maxIndexedDay(db, "VersionSummary")
+	log.Println("Aggregating VersionSummary data since", since)
+	rows, err := aggregateVersionSummary(db, since)
+	if err != nil {
+		log.Fatalln("aggregate:", err)
+	}
+	log.Println("Inserted", rows, "rows")
+
+	log.Println("Aggregating UserMovement data")
+	rows, err = aggregateUserMovement(db)
+	if err != nil {
+		log.Fatalln("aggregate:", err)
+	}
+	log.Println("Inserted", rows, "rows")
+
+	log.Println("Aggregating Performance data")
+	since = maxIndexedDay(db, "Performance")
+	rows, err = aggregatePerformance(db, since)
+	if err != nil {
+		log.Fatalln("aggregate:", err)
+	}
+	log.Println("Inserted", rows, "rows")
+
+	log.Println("Aggregating BlockStats data")
+	since = maxIndexedDay(db, "BlockStats")
+	rows, err = aggregateBlockStats(db, since)
+	if err != nil {
+		log.Fatalln("aggregate:", err)
+	}
+	log.Println("Inserted", rows, "rows")
+}
+
+func sleepUntilNext(intv, margin time.Duration) {
+	now := time.Now().UTC()
+	next := now.Truncate(intv).Add(intv).Add(margin)
+	log.Println("Sleeping until", next)
+	time.Sleep(next.Sub(now))
+}
+
+func setupDB(db *sql.DB) error {
+	_, err := db.Exec(`CREATE TABLE IF NOT EXISTS VersionSummary (
+		Day TIMESTAMP NOT NULL,
+		Version VARCHAR(8) NOT NULL,
+		Count INTEGER NOT NULL
+	)`)
+	if err != nil {
+		return err
+	}
+
+	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS UserMovement (
+		Day TIMESTAMP NOT NULL,
+		Added INTEGER NOT NULL,
+		Bounced INTEGER NOT NULL,
+		Removed INTEGER NOT NULL
+	)`)
+	if err != nil {
+		return err
+	}
+
+	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Performance (
+		Day TIMESTAMP NOT NULL,
+		TotFiles INTEGER NOT NULL,
+		TotMiB INTEGER NOT NULL,
+		SHA256Perf DOUBLE PRECISION NOT NULL,
+		MemorySize INTEGER NOT NULL,
+		MemoryUsageMiB INTEGER NOT NULL
+	)`)
+	if err != nil {
+		return err
+	}
+
+	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS BlockStats (
+		Day TIMESTAMP NOT NULL,
+		Reports INTEGER NOT NULL,
+		Total INTEGER NOT NULL,
+		Renamed INTEGER NOT NULL,
+		Reused INTEGER NOT NULL,
+		Pulled INTEGER NOT NULL,
+		CopyOrigin INTEGER NOT NULL,
+		CopyOriginShifted INTEGER NOT NULL,
+		CopyElsewhere INTEGER NOT NULL
+	)`)
+	if err != nil {
+		return err
+	}
+
+	var t string
+
+	row := db.QueryRow(`SELECT 'UniqueDayVersionIndex'::regclass`)
+	if err := row.Scan(&t); err != nil {
+		_, err = db.Exec(`CREATE UNIQUE INDEX UniqueDayVersionIndex ON VersionSummary (Day, Version)`)
+	}
+
+	row = db.QueryRow(`SELECT 'VersionDayIndex'::regclass`)
+	if err := row.Scan(&t); err != nil {
+		_, err = db.Exec(`CREATE INDEX VersionDayIndex ON VersionSummary (Day)`)
+	}
+
+	row = db.QueryRow(`SELECT 'MovementDayIndex'::regclass`)
+	if err := row.Scan(&t); err != nil {
+		_, err = db.Exec(`CREATE INDEX MovementDayIndex ON UserMovement (Day)`)
+	}
+
+	row = db.QueryRow(`SELECT 'PerformanceDayIndex'::regclass`)
+	if err := row.Scan(&t); err != nil {
+		_, err = db.Exec(`CREATE INDEX PerformanceDayIndex ON Performance (Day)`)
+	}
+
+	row = db.QueryRow(`SELECT 'BlockStatsDayIndex'::regclass`)
+	if err := row.Scan(&t); err != nil {
+		_, err = db.Exec(`CREATE INDEX BlockStatsDayIndex ON BlockStats (Day)`)
+	}
+
+	return err
+}
+
+func maxIndexedDay(db *sql.DB, table string) time.Time {
+	var t time.Time
+	row := db.QueryRow("SELECT MAX(Day) FROM " + table)
+	err := row.Scan(&t)
+	if err != nil {
+		return time.Time{}
+	}
+	return t
+}
+
+func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) {
+	res, err := db.Exec(`INSERT INTO VersionSummary (
+	SELECT
+		DATE_TRUNC('day', Received) AS Day,
+		SUBSTRING(Version FROM '^v\d.\d+') AS Ver,
+		COUNT(*) AS Count
+		FROM Reports
+		WHERE
+			DATE_TRUNC('day', Received) > $1
+			AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
+			AND Version like 'v_.%'
+		GROUP BY Day, Ver
+		);
+	`, since)
+	if err != nil {
+		return 0, err
+	}
+
+	return res.RowsAffected()
+}
+
+func aggregateUserMovement(db *sql.DB) (int64, error) {
+	rows, err := db.Query(`SELECT
+		DATE_TRUNC('day', Received) AS Day,
+		UniqueID
+		FROM Reports
+		WHERE
+			DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
+			AND Version like 'v_.%'
+		ORDER BY Day
+	`)
+	if err != nil {
+		return 0, err
+	}
+	defer rows.Close()
+
+	firstSeen := make(map[string]time.Time)
+	lastSeen := make(map[string]time.Time)
+	var minTs time.Time
+	minTs = minTs.In(time.UTC)
+
+	for rows.Next() {
+		var ts time.Time
+		var id string
+		if err := rows.Scan(&ts, &id); err != nil {
+			return 0, err
+		}
+
+		if minTs.IsZero() {
+			minTs = ts
+		}
+		if _, ok := firstSeen[id]; !ok {
+			firstSeen[id] = ts
+		}
+		lastSeen[id] = ts
+	}
+
+	type sumRow struct {
+		day     time.Time
+		added   int
+		removed int
+		bounced int
+	}
+	var sumRows []sumRow
+	for t := minTs; t.Before(time.Now().Truncate(24 * time.Hour)); t = t.AddDate(0, 0, 1) {
+		var added, removed, bounced int
+		old := t.Before(time.Now().AddDate(0, 0, -30))
+		for id, first := range firstSeen {
+			last := lastSeen[id]
+			if first.Equal(t) && last.Equal(t) && old {
+				bounced++
+				continue
+			}
+			if first.Equal(t) {
+				added++
+			}
+			if last == t && old {
+				removed++
+			}
+		}
+		sumRows = append(sumRows, sumRow{t, added, removed, bounced})
+	}
+
+	tx, err := db.Begin()
+	if err != nil {
+		return 0, err
+	}
+	if _, err := tx.Exec("DELETE FROM UserMovement"); err != nil {
+		tx.Rollback()
+		return 0, err
+	}
+	for _, r := range sumRows {
+		if _, err := tx.Exec("INSERT INTO UserMovement (Day, Added, Removed, Bounced) VALUES ($1, $2, $3, $4)", r.day, r.added, r.removed, r.bounced); err != nil {
+			tx.Rollback()
+			return 0, err
+		}
+	}
+
+	return int64(len(sumRows)), tx.Commit()
+}
+
+func aggregatePerformance(db *sql.DB, since time.Time) (int64, error) {
+	res, err := db.Exec(`INSERT INTO Performance (
+	SELECT
+		DATE_TRUNC('day', Received) AS Day,
+		AVG(TotFiles) As TotFiles,
+		AVG(TotMiB) As TotMiB,
+		AVG(SHA256Perf) As SHA256Perf,
+		AVG(MemorySize) As MemorySize,
+		AVG(MemoryUsageMiB) As MemoryUsageMiB
+		FROM Reports
+		WHERE
+			DATE_TRUNC('day', Received) > $1
+			AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
+			AND Version like 'v_.%'
+		GROUP BY Day
+		);
+	`, since)
+	if err != nil {
+		return 0, err
+	}
+
+	return res.RowsAffected()
+}
+
+func aggregateBlockStats(db *sql.DB, since time.Time) (int64, error) {
+	// Filter out anything prior 0.14.41 as that has sum aggregations which
+	// made no sense.
+	res, err := db.Exec(`INSERT INTO BlockStats (
+	SELECT
+		DATE_TRUNC('day', Received) AS Day,
+		COUNT(1) As Reports,
+		SUM(BlocksTotal) AS Total,
+		SUM(BlocksRenamed) AS Renamed,
+		SUM(BlocksReused) AS Reused,
+		SUM(BlocksPulled) AS Pulled,
+		SUM(BlocksCopyOrigin) AS CopyOrigin,
+		SUM(BlocksCopyOriginShifted) AS CopyOriginShifted,
+		SUM(BlocksCopyElsewhere) AS CopyElsewhere
+		FROM Reports
+		WHERE
+			DATE_TRUNC('day', Received) > $1
+			AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
+			AND ReportVersion = 3
+			AND Version like 'v_.%'
+			AND Version NOT LIKE 'v0.14.40%'
+			AND Version NOT LIKE 'v0.14.39%'
+			AND Version NOT LIKE 'v0.14.38%'
+		GROUP BY Day
+	);
+	`, since)
+	if err != nil {
+		return 0, err
+	}
+
+	return res.RowsAffected()
+}

cmd/ursrv/analytics.go

@@ -0,0 +1,240 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+type analytic struct {
+	Key        string
+	Count      int
+	Percentage float64
+	Items      []analytic `json:",omitempty"`
+}
+
+type analyticList []analytic
+
+func (l analyticList) Less(a, b int) bool {
+	if l[a].Key == "Others" {
+		return false
+	}
+	if l[b].Key == "Others" {
+		return true
+	}
+	return l[b].Count < l[a].Count // inverse
+}
+
+func (l analyticList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+
+func (l analyticList) Len() int {
+	return len(l)
+}
+
+// Returns a list of frequency analytics for a given list of strings.
+func analyticsFor(ss []string, cutoff int) []analytic {
+	m := make(map[string]int)
+	t := 0
+	for _, s := range ss {
+		m[s]++
+		t++
+	}
+
+	l := make([]analytic, 0, len(m))
+	for k, c := range m {
+		l = append(l, analytic{
+			Key:        k,
+			Count:      c,
+			Percentage: 100 * float64(c) / float64(t),
+		})
+	}
+
+	sort.Sort(analyticList(l))
+
+	if cutoff > 0 && len(l) > cutoff {
+		c := 0
+		for _, i := range l[cutoff:] {
+			c += i.Count
+		}
+		l = append(l[:cutoff], analytic{
+			Key:        "Others",
+			Count:      c,
+			Percentage: 100 * float64(c) / float64(t),
+		})
+	}
+
+	return l
+}
+
+// Find the points at which certain penetration levels are met
+func penetrationLevels(as []analytic, points []float64) []analytic {
+	sort.Slice(as, func(a, b int) bool {
+		return versionLess(as[b].Key, as[a].Key)
+	})
+
+	var res []analytic
+
+	idx := 0
+	sum := 0.0
+	for _, a := range as {
+		sum += a.Percentage
+		if sum >= points[idx] {
+			a.Count = int(points[idx])
+			a.Percentage = sum
+			res = append(res, a)
+			idx++
+			if idx == len(points) {
+				break
+			}
+		}
+	}
+	return res
+}
+
+func statsForInts(data []int) [4]float64 {
+	var res [4]float64
+	if len(data) == 0 {
+		return res
+	}
+
+	sort.Ints(data)
+	res[0] = float64(data[int(float64(len(data))*0.05)])
+	res[1] = float64(data[len(data)/2])
+	res[2] = float64(data[int(float64(len(data))*0.95)])
+	res[3] = float64(data[len(data)-1])
+	return res
+}
+
+func statsForFloats(data []float64) [4]float64 {
+	var res [4]float64
+	if len(data) == 0 {
+		return res
+	}
+
+	sort.Float64s(data)
+	res[0] = data[int(float64(len(data))*0.05)]
+	res[1] = data[len(data)/2]
+	res[2] = data[int(float64(len(data))*0.95)]
+	res[3] = data[len(data)-1]
+	return res
+}
+
+func group(by func(string) string, as []analytic, perGroup int) []analytic {
+	var res []analytic
+
+next:
+	for _, a := range as {
+		group := by(a.Key)
+		for i := range res {
+			if res[i].Key == group {
+				res[i].Count += a.Count
+				res[i].Percentage += a.Percentage
+				if len(res[i].Items) < perGroup {
+					res[i].Items = append(res[i].Items, a)
+				}
+				continue next
+			}
+		}
+		res = append(res, analytic{
+			Key:        group,
+			Count:      a.Count,
+			Percentage: a.Percentage,
+			Items:      []analytic{a},
+		})
+	}
+
+	sort.Sort(analyticList(res))
+	return res
+}
+
+func byVersion(s string) string {
+	parts := strings.Split(s, ".")
+	if len(parts) >= 2 {
+		return strings.Join(parts[:2], ".")
+	}
+	return s
+}
+
+func byPlatform(s string) string {
+	parts := strings.Split(s, "-")
+	if len(parts) >= 2 {
+		return parts[0]
+	}
+	return s
+}
+
+var numericGoVersion = regexp.MustCompile(`^go[0-9]\.[0-9]+`)
+
+func byCompiler(s string) string {
+	if m := numericGoVersion.FindString(s); m != "" {
+		return m
+	}
+	return "Other"
+}
+
+func versionLess(a, b string) bool {
+	arel, apre := versionParts(a)
+	brel, bpre := versionParts(b)
+
+	minlen := len(arel)
+	if l := len(brel); l < minlen {
+		minlen = l
+	}
+
+	for i := 0; i < minlen; i++ {
+		if arel[i] != brel[i] {
+			return arel[i] < brel[i]
+		}
+	}
+
+	// Longer version is newer, when the preceding parts are equal
+	if len(arel) != len(brel) {
+		return len(arel) < len(brel)
+	}
+
+	if apre != bpre {
+		// "(+dev)" versions are ahead
+		if apre == plusStr {
+			return false
+		}
+		if bpre == plusStr {
+			return true
+		}
+		return apre < bpre
+	}
+
+	// don't actually care how the prerelease stuff compares for our purposes
+	return false
+}
+
+// Split a version as returned from transformVersion into parts.
+// "1.2.3-beta.2" -> []int{1, 2, 3}, "beta.2"}
+func versionParts(v string) ([]int, string) {
+	parts := strings.SplitN(v[1:], " ", 2) // " (+dev)" versions
+	if len(parts) == 1 {
+		parts = strings.SplitN(parts[0], "-", 2) // "-rc.1" type versions
+	}
+	fields := strings.Split(parts[0], ".")
+
+	release := make([]int, len(fields))
+	for i, s := range fields {
+		v, _ := strconv.Atoi(s)
+		release[i] = v
+	}
+
+	var prerelease string
+	if len(parts) > 1 {
+		prerelease = parts[1]
+	}
+
+	return release, prerelease
+}

cmd/ursrv/compiler_test.go

@@ -0,0 +1,31 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import "testing"
+
+func TestCompilerRe(t *testing.T) {
+	tests := [][3]string{
+		{`syncthing v0.11.0 (xgcc (Ubuntu 4.9.3-0ubuntu4) 4.9.3 linux-amd64 default) niklas@Niklas-Netbook 2015-04-26 13:15:08 UTC`, "xgcc (Ubuntu 4.9.3-0ubuntu4) 4.9.3", "niklas@Niklas-Netbook"},
+		{`syncthing v0.12.0-rc5 "Beryllium Bedbug" (go1.4.2 linux-arm android) unknown-user@Felix-T420 2015-10-22 18:32:15 UTC`, "go1.4.2", "unknown-user@Felix-T420"},
+		{`syncthing v0.13.0-beta.0+39-ge267bf3 "Copper Cockroach" (go1.4.2 linux-amd64) portage@slevermann.de 2016-01-20 08:41:52 UTC`, "go1.4.2", "portage@slevermann.de"},
+	}
+
+	for _, tc := range tests {
+		m := compilerRe.FindStringSubmatch(tc[0])
+		if len(m) != 3 {
+			t.Errorf("Regexp didn't match %q", tc[0])
+			continue
+		}
+		if m[1] != tc[1] {
+			t.Errorf("Compiler %q != %q", m[1], tc[1])
+		}
+		if m[2] != tc[2] {
+			t.Errorf("Builder %q != %q", m[2], tc[2])
+		}
+	}
+}

cmd/ursrv/formatting.go

@@ -0,0 +1,131 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+)
+
+type NumberType int
+
+const (
+	NumberMetric NumberType = iota
+	NumberBinary
+	NumberDuration
+)
+
+func number(ntype NumberType, v float64) string {
+	switch ntype {
+	case NumberMetric:
+		return metric(v)
+	case NumberDuration:
+		return duration(v)
+	case NumberBinary:
+		return binary(v)
+	default:
+		return metric(v)
+	}
+}
+
+type suffix struct {
+	Suffix     string
+	Multiplier float64
+}
+
+var metricSuffixes = []suffix{
+	{"G", 1e9},
+	{"M", 1e6},
+	{"k", 1e3},
+}
+
+var binarySuffixes = []suffix{
+	{"Gi", 1 << 30},
+	{"Mi", 1 << 20},
+	{"Ki", 1 << 10},
+}
+
+var durationSuffix = []suffix{
+	{"year", 365 * 24 * 60 * 60},
+	{"month", 30 * 24 * 60 * 60},
+	{"day", 24 * 60 * 60},
+	{"hour", 60 * 60},
+	{"minute", 60},
+	{"second", 1},
+}
+
+func metric(v float64) string {
+	return withSuffix(v, metricSuffixes, false)
+}
+
+func binary(v float64) string {
+	return withSuffix(v, binarySuffixes, false)
+}
+
+func duration(v float64) string {
+	return withSuffix(v, durationSuffix, true)
+}
+
+func withSuffix(v float64, ps []suffix, pluralize bool) string {
+	for _, p := range ps {
+		if v >= p.Multiplier {
+			suffix := p.Suffix
+			if pluralize && v/p.Multiplier != 1.0 {
+				suffix += "s"
+			}
+			// If the number only has decimal zeroes, strip em off.
+			num := strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.1f", v/p.Multiplier), "0"), ".")
+			return fmt.Sprintf("%s %s", num, suffix)
+		}
+	}
+	return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.1f", v), "0"), ".")
+}
+
+// commatize returns a number with sep as thousands separators. Handles
+// integers and plain floats.
+func commatize(sep, s string) string {
+	// If no dot, don't do anything.
+	if !strings.ContainsRune(s, '.') {
+		return s
+	}
+	var b bytes.Buffer
+	fs := strings.SplitN(s, ".", 2)
+
+	l := len(fs[0])
+	for i := range fs[0] {
+		b.Write([]byte{s[i]})
+		if i < l-1 && (l-i)%3 == 1 {
+			b.WriteString(sep)
+		}
+	}
+
+	if len(fs) > 1 && len(fs[1]) > 0 {
+		b.WriteString(".")
+		b.WriteString(fs[1])
+	}
+
+	return b.String()
+}
+
+func proportion(m map[string]int, count int) float64 {
+	total := 0
+	isMax := true
+	for _, n := range m {
+		total += n
+		if n > count {
+			isMax = false
+		}
+	}
+	pct := (100 * float64(count)) / float64(total)
+	// To avoid rounding errors in the template, surpassing 100% and breaking
+	// the progress bars.
+	if isMax && len(m) > 1 && count != total {
+		pct -= 0.01
+	}
+	return pct
+}

cmd/ursrv/static/index.html

@@ -0,0 +1,656 @@
+<!DOCTYPE html>
+<!--
+Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
+Use of this source code is governed by an MIT-style license that can be
+found in the LICENSE file.
+-->
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <meta name="description" content="">
+  <meta name="author" content="">
+  <link rel="shortcut icon" href="static/assets/img/favicon.png">
+
+  <title>Syncthing Usage Reports</title>
+  <link href="static/bootstrap/css/bootstrap.min.css" rel="stylesheet">
+  <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
+  <script type="text/javascript" src="static/bootstrap/js/bootstrap.min.js"></script>
+  <script type="text/javascript" src="https://maps.googleapis.com/maps/api/js?libraries=visualization&key=AIzaSyDk5WJ8s7ueLKb99X5DbQ-vkWtPDAKqYs0"></script>
+  <style type="text/css">
+    body {
+      margin: 40px;
+      font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
+    }
+    tr.main td {
+      font-weight: bold;
+    }
+    tr.child td.first {
+      padding-left: 2em;
+    }
+    .progress-bar {
+      overflow:hidden;
+      white-space:nowrap;
+      text-overflow: ellipsis;
+    }
+  </style>
+  <script type="text/javascript"
+          src="https://www.google.com/jsapi?autoload={
+            'modules':[{
+              'name':'visualization',
+              'version':'1',
+              'packages':['corechart']
+            }]
+          }"></script>
+
+  <script type="text/javascript">
+    google.setOnLoadCallback(drawVersionChart);
+    google.setOnLoadCallback(drawMovementChart);
+    google.setOnLoadCallback(drawBlockStatsChart);
+    google.setOnLoadCallback(drawPerformanceCharts);
+    google.setOnLoadCallback(drawHeatMap);
+
+    function drawVersionChart() {
+      var jsonData = $.ajax({url: "summary.json", dataType:"json", async: false}).responseText;
+      var rows = JSON.parse(jsonData);
+
+      var data = new google.visualization.DataTable();
+      data.addColumn('date', 'Day');
+      for (var i = 1; i < rows[0].length; i++){
+        data.addColumn('number', rows[0][i]);
+      }
+      for (var i = 1; i < rows.length; i++){
+        rows[i][0] = new Date(rows[i][0]);
+        data.addRow(rows[i]);
+      };
+
+      var options = {
+        legend: { position: 'bottom', alignment: 'center' },
+        isStacked: true,
+        colors: ['rgb(102,194,165)','rgb(252,141,98)','rgb(141,160,203)','rgb(231,138,195)','rgb(166,216,84)','rgb(255,217,47)'],
+        chartArea: {left: 80, top: 20, width: '1020', height: '300'},
+      };
+
+      var chart = new google.visualization.AreaChart(document.getElementById('versionChart'));
+      chart.draw(data, options);
+    }
+
+    function drawMovementChart() {
+      var jsonData = $.ajax({url: "movement.json", dataType:"json", async: false}).responseText;
+      var rows = JSON.parse(jsonData);
+
+      var data = new google.visualization.DataTable();
+      data.addColumn('date', 'Day');
+      for (var i = 1; i < rows[0].length; i++){
+        data.addColumn('number', rows[0][i]);
+      }
+
+      for (var i = 1; i < rows.length; i++){
+        rows[i][0] = new Date(rows[i][0]);
+        if (rows[i][1] > 500) {
+          rows[i][1] = null;
+        }
+        if (rows[i][2] < -500) {
+          rows[i][2] = null;
+        }
+        data.addRow(rows[i]);
+      };
+
+      var options = {
+        legend: { position: 'bottom', alignment: 'center' },
+        colors: ['rgb(102,194,165)','rgb(252,141,98)','rgb(141,160,203)','rgb(231,138,195)','rgb(166,216,84)','rgb(255,217,47)'],
+        chartArea: {left: 80, top: 20, width: '1020', height: '300'},
+      };
+
+      var chart = new google.visualization.AreaChart(document.getElementById('movementChart'));
+      chart.draw(data, options);
+    }
+
+    function formatGibibytes(gibibytes, decimals) {
+      if(gibibytes == 0) return '0 GiB';
+      var k = 1024,
+        dm = decimals || 2,
+        sizes = ['GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'],
+        i = Math.floor(Math.log(gibibytes) / Math.log(k));
+      if (i < 0) {
+        sizes = 'MiB';
+      } else {
+        sizes = sizes[i];
+      }
+      return parseFloat((gibibytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes;
+    }
+
+
+    function drawBlockStatsChart() {
+      var jsonData = $.ajax({url: "blockstats.json", dataType:"json", async: false}).responseText;
+      var rows = JSON.parse(jsonData);
+
+      var data = new google.visualization.DataTable();
+      data.addColumn('date', 'Day');
+      for (var i = 1; i < rows[0].length; i++){
+        data.addColumn('number', rows[0][i]);
+      }
+
+      var totals = [0, 0, 0, 0, 0, 0];
+      for (var i = 1; i < rows.length; i++){
+        rows[i][0] = new Date(rows[i][0]);
+        for (var j = 2; j < rows[i].length; j++) {
+          totals[j-2] += rows[i][j];
+        }
+        data.addRow(rows[i]);
+      };
+
+      var totalTotals = totals.reduce(function(a, b) { return a + b; }, 0);
+
+      if (totalTotals > 0) {
+        var content = "<table class='table'>\n"
+        for (var j = 2; j < rows[0].length; j++) {
+          content += "<tr><td><b>" + rows[0][j].replace(' (GiB)', '') + "</b></td><td>" + formatGibibytes(totals[j-2].toFixed(2)) + " (" + ((100*totals[j-2])/totalTotals).toFixed(2) +"%)</td></tr>\n";
+        }
+        content += "</table>";
+        document.getElementById("data-to-date").innerHTML = content;
+      } else {
+        // No data, hide it.
+        document.getElementById("block-stats").outerHTML  = "";
+        return;
+      }
+
+      var options = {
+        focusTarget: 'category',
+        vAxes: {0: {}, 1: {}},
+        series: {0: {type: 'line', targetAxisIndex:1}},
+        isStacked: true,
+        legend: {position: 'none'},
+        colors: ['rgb(102,194,165)','rgb(252,141,98)','rgb(141,160,203)','rgb(231,138,195)','rgb(166,216,84)','rgb(255,217,47)'],
+        chartArea: {left: 80, top: 20, width: '1020', height: '300'},
+      };
+
+      var chart = new google.visualization.AreaChart(document.getElementById('blockStatsChart'));
+      chart.draw(data, options);
+    }
+
+    function drawPerformanceCharts() {
+      var jsonData = $.ajax({url: "/performance.json", dataType:"json", async: false}).responseText;
+      var rows = JSON.parse(jsonData);
+      for (var i = 1; i < rows.length; i++){
+        rows[i][0] = new Date(rows[i][0]);
+      }
+
+      drawChart(rows, 1, 'Total Number of Files', 'totFilesChart', 1e6, 1);
+      drawChart(rows, 2, 'Total Folder Size (GiB)', 'totMiBChart', 1e6, 1024);
+      drawChart(rows, 3, 'Hash Performance (MiB/s)', 'hashPerfChart', 1000, 1);
+      drawChart(rows, 4, 'System RAM Size (GiB)', 'memSizeChart', 1e6, 1024);
+      drawChart(rows, 5, 'Memory Usage (MiB)', 'memUsageChart', 250, 1);
+    }
+
+    function drawChart(rows, index, title, id, cutoff, divisor) {
+      var data = new google.visualization.DataTable();
+      data.addColumn('date', 'Day');
+      data.addColumn('number', title);
+
+      var row;
+      for (var i = 1; i < rows.length; i++){
+          row = [rows[i][0], rows[i][index] / divisor];
+        if (row[1] > cutoff) {
+          row[1] = null;
+        }
+        data.addRow(row);
+      }
+
+      var options = {
+        legend: { position: 'bottom', alignment: 'center' },
+        colors: ['rgb(102,194,165)','rgb(252,141,98)','rgb(141,160,203)','rgb(231,138,195)','rgb(166,216,84)','rgb(255,217,47)'],
+        chartArea: {left: 80, top: 20, width: '1020', height: '300'},
+        vAxes: {0: {minValue: 0}},
+      };
+
+      var chart = new google.visualization.LineChart(document.getElementById(id));
+      chart.draw(data, options);
+    }
+
+    var locations = [];
+    {{range $location, $weight := .locations}}
+    locations.push({location: new google.maps.LatLng({{- $location.Latitude -}}, {{- $location.Longitude -}}), weight: {{- $weight -}}});
+    {{- end}}
+
+    function drawHeatMap() {
+      if (locations.length == 0) {
+        return;
+      }
+      var mapBounds = new google.maps.LatLngBounds();
+      var map = new google.maps.Map(document.getElementById('map'), {
+        zoom: 1,
+        mapTypeId: google.maps.MapTypeId.ROADMAP
+      });
+      var heatmap = new google.maps.visualization.HeatmapLayer({
+        data: locations
+      });
+      heatmap.set('radius', 10);
+      heatmap.set('maxIntensity', 20);
+      heatmap.set('gradient', [
+        'rgba(0, 255, 255, 0)',
+        'rgba(0, 255, 255, 1)',
+        'rgba(0, 191, 255, 1)',
+        'rgba(0, 127, 255, 1)',
+        'rgba(0, 63, 255, 1)',
+        'rgba(0, 0, 255, 1)',
+        'rgba(0, 0, 223, 1)',
+        'rgba(0, 0, 191, 1)',
+        'rgba(0, 0, 159, 1)',
+        'rgba(0, 0, 127, 1)',
+        'rgba(63, 0, 91, 1)',
+        'rgba(127, 0, 63, 1)',
+        'rgba(191, 0, 31, 1)',
+        'rgba(255, 0, 0, 1)'
+      ]);
+      heatmap.setMap(map);
+      for (var x = 0; x < locations.length; x++) {
+        mapBounds.extend(locations[x].location);
+      }
+      map.fitBounds(mapBounds);
+      if (locations.length == 1) {
+        map.setZoom(13);
+      }
+    }
+  </script>
+</head>
+
+<body>
+  <div class="container">
+    <div class="row">
+      <div class="col-md-12">
+        <h1>Syncthing Usage Data</h1>
+
+        <h4 id="active-users">Active Users per Day and Version</h4>
+        <p>
+          This is the total number of unique users with reporting enabled, per day. Area color represents the major version.
+        </p>
+        <div class="img-thumbnail" id="versionChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+
+        <h4 id="joining-leaving">Users Joining and Leaving per Day</h4>
+        <p>
+          This is the total number of unique users joining and leaving per day. A user is counted as "joined" on first the day their unique ID is seen, and as "left" on the last day the unique ID was seen before a two weeks or longer absence. "Bounced" refers to users who joined and left on the same day.
+        </p>
+        <div class="img-thumbnail" id="movementChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+        <p class="text-muted">
+         Reappearance of users cause the "left" data to shrink retroactively.
+        </p>
+        <div id="block-stats">
+          <h4>Data Transfers per Day</h4>
+          <p>
+            This is total data transferred per day. Also shows how much data was saved (not transferred) by each of the methods syncthing uses.
+          </p>
+          <div class="img-thumbnail" id="blockStatsChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+          <h4 id="totals-to-date">Totals to date</h4>
+          <p id="data-to-date">
+            No data
+          </p>
+        </div>
+
+        <h4 id="metrics">Usage Metrics</h4>
+        <p>
+          This is the aggregated usage report data for the last 24 hours. Data based on <b>{{.nodes}}</b> devices that have reported in.
+        </p>
+
+        {{if .locations}}
+        <div class="img-thumbnail" id="map" style="width: 1130px; height: 400px; padding: 10px;"></div>
+        <p class="text-muted">
+         Heatmap max intensity is capped at 20 reports within a location.
+        </p>
+        <div class="panel panel-default">
+          <div class="panel-heading">
+            <h4 class="panel-title">
+              <a data-toggle="collapse" href="#collapseTwo">Break down per country</a>
+            </h4>
+          </div>
+          <div id="collapseTwo" class="panel-collapse collapse">
+            <div class="panel-body">
+              <div class="row">
+                <div class="col-md-6">
+                  <table class="table table-striped">
+                    <tbody>
+                      {{range .contries | slice 2 1}}
+                      <tr>
+                        <td style="width: 45%">{{.Key}}</td>
+                        <td style="width: 5%" class="text-right">{{if ge .Pct 10.0}}{{.Pct | printf "%.0f"}}{{else if ge .Pct 1.0}}{{.Pct | printf "%.01f"}}{{else}}{{.Pct | printf "%.02f"}}{{end}}%</td>
+                        <td style="width: 5%" class="text-right">{{.Count}}</td>
+                        <td>
+                            <div class="progress-bar" role="progressbar" aria-valuenow="{{.Pct | printf "%.02f"}}" aria-valuemin="0" aria-valuemax="100" style="width: {{.Pct | printf "%.02f"}}%; height:20px"></div>
+                        </td>
+                      </tr>
+                      {{end}}
+                    </tbody>
+                  </table>
+                </div>
+                <div class="col-md-6">
+                  <table class="table table-striped">
+                    <tbody>
+                      {{range .contries | slice 2 2}}
+                      <tr>
+                        <td style="width: 45%">{{.Key}}</td>
+                        <td style="width: 5%" class="text-right">{{if ge .Pct 10.0}}{{.Pct | printf "%.0f"}}{{else if ge .Pct 1.0}}{{.Pct | printf "%.01f"}}{{else}}{{.Pct | printf "%.02f"}}{{end}}%</td>
+                        <td style="width: 5%" class="text-right">{{.Count}}</td>
+                        <td>
+                            <div class="progress-bar" role="progressbar" aria-valuenow="{{.Pct | printf "%.02f"}}" aria-valuemin="0" aria-valuemax="100" style="width: {{.Pct | printf "%.02f"}}%; height:20px"></div>
+                        </td>
+                      </tr>
+                      {{end}}
+                    </tbody>
+                  </table>
+                </div>
+              </div>
+            </div>
+          </div>
+        </div>
+        {{end}}
+        <table class="table table-striped">
+          <thead>
+            <tr>
+              <th></th>
+              <th colspan="4" class="text-center">
+                <a href="https://en.wikipedia.org/wiki/Percentile">Percentile</a>
+              </th>
+            </tr>
+            <tr>
+              <th></th>
+              <th class="text-right">5%</th>
+              <th class="text-right">50%</th>
+              <th class="text-right">95%</th>
+              <th class="text-right">100%</th>
+            </tr>
+          </thead>
+          <tbody>
+            {{range .categories}}
+            <tr>
+              <td>{{.Descr}}</td>
+              <td class="text-right">{{index .Values 0 | number .Type | commatize " "}}{{.Unit}}</td>
+              <td class="text-right">{{index .Values 1 | number .Type | commatize " "}}{{.Unit}}</td>
+              <td class="text-right">{{index .Values 2 | number .Type | commatize " "}}{{.Unit}}</td>
+              <td class="text-right">{{index .Values 3 | number .Type | commatize " "}}{{.Unit}}</td>
+            </tr>
+            {{end}}
+          </tbody>
+        </table>
+      </div>
+    </div>
+
+    <div class="row">
+      <div class="col-md-6">
+        <table class="table table-striped">
+          <thead>
+            <tr>
+              <th>Version</th><th class="text-right">Devices</th><th class="text-right">Share</th>
+            </tr>
+          </thead>
+          <tbody>
+            {{range .versions}}
+              {{if gt .Percentage 0.1}}
+                <tr class="main">
+                  <td>{{.Key}}</td>
+                  <td class="text-right">{{.Count}}</td>
+                  <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
+                </tr>
+                {{range .Items}}
+                  {{if gt .Percentage 0.1}}
+                    <tr class="child">
+                      <td class="first">{{.Key}}</td>
+                      <td class="text-right">{{.Count}}</td>
+                      <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
+                    </tr>
+                  {{end}}
+                {{end}}
+              {{end}}
+            {{end}}
+          </tbody>
+        </table>
+        <table class="table table-striped">
+          <thead>
+            <tr>
+                <th>Penetration Level</th>
+                <th>Version</th>
+              <th class="text-right">Actual</th>
+            </tr>
+          </thead>
+          <tbody>
+            {{range .versionPenetrations}}
+            <tr>
+                <td>{{.Count}}%</td>
+                <td>&ge; {{.Key}}</td>
+              <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
+            </tr>
+            {{end}}
+          </tbody>
+        </table>
+      </div>
+
+      <div class="col-md-6">
+        <table class="table table-striped">
+          <thead>
+            <tr>
+              <th>Platform</th>
+              <th class="text-right">Devices</th>
+              <th class="text-right">Share</th>
+            </tr>
+          </thead>
+          <tbody>
+            {{range .platforms}}
+              <tr class="main">
+                <td>{{.Key}}</td>
+                <td class="text-right">{{.Count}}</td>
+                <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
+              </tr>
+              {{range .Items}}
+                <tr class="child">
+                  <td class="first">{{.Key}}</td>
+                  <td class="text-right">{{.Count}}</td>
+                  <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
+                </tr>
+              {{end}}
+            {{end}}
+          </tbody>
+        </table>
+      </div>
+
+    </div>
+    <div class="row">
+
+      <div class="col-md-6">
+        <table class="table table-striped">
+          <thead>
+            <tr>
+              <th>Compiler</th>
+              <th class="text-right">Devices</th>
+              <th class="text-right">Share</th>
+            </tr>
+          </thead>
+          <tbody>
+            {{range .compilers}}
+              <tr class="main">
+                <td>{{.Key}}</td>
+                <td class="text-right">{{.Count}}</td>
+                <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
+              </tr>
+              {{range .Items}}
+                {{if or (gt .Percentage 0.1) (eq .Key "Others")}}
+                  <tr class="child">
+                    <td class="first">{{.Key}}</td>
+                    <td class="text-right">{{.Count}}</td>
+                    <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
+                  </tr>
+                {{end}}
+              {{end}}
+            {{end}}
+          </tbody>
+        </table>
+      </div>
+
+      <div class="col-md-6">
+        <table class="table table-striped">
+          <thead>
+            <tr>
+              <th>Builder</th>
+              <th class="text-right">Devices</th>
+              <th class="text-right">Share</th>
+            </tr>
+          </thead>
+          <tbody>
+            {{range .builders}}
+            <tr>
+              <td>{{.Key}}</td>
+              <td class="text-right">{{.Count}}</td>
+              <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
+            </tr>
+            {{end}}
+          </tbody>
+        </table>
+      </div>
+
+      <div class="col-md-6">
+        <table class="table table-striped">
+          <thead>
+            <tr>
+              <th>Distribution Channel</th>
+              <th class="text-right">Devices</th>
+              <th class="text-right">Share</th>
+            </tr>
+          </thead>
+          <tbody>
+            {{range .distributions}}
+            <tr>
+              <td>{{.Key}}</td>
+              <td class="text-right">{{.Count}}</td>
+              <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
+            </tr>
+            {{end}}
+          </tbody>
+        </table>
+      </div>
+
+    </div>
+
+    <div class="row">
+        <div class="col-md-12">
+            <h4 id="features">Feature Usage</h4>
+            <p>
+                The following lists feature usage. Some features are reported per report, some are per sum of units within report (eg. devices with static addresses among all known devices per report).
+                Currently there are <b>{{.versionNodes.v2}}</b> devices reporting for version 2 and <b>{{.versionNodes.v3}}</b> for version 3.
+            </p>
+        </div>
+    </div>
+
+
+    <div class="row">
+    {{$i := counter}}
+    {{range $featureName := .featureOrder}}
+      {{$featureValues := index $.features $featureName }}
+      {{if $i.DrawTwoDivider}}
+        </div>
+        <div class="row">
+      {{end}}
+      {{ $i.Increment }}
+      <div class="col-md-6">
+          <table class="table table-striped">
+              <thead><tr>
+                  <th>{{$featureName}} Features</th><th colspan="2" class="text-center">Usage</th>
+              </tr></thead>
+              <tbody>
+                  {{range $featureValues}}
+                  <tr>
+                      <td style="width: 50%">{{.Key}} ({{.Version}})</td>
+                      <td style="width: 10%" class="text-right">{{if ge .Pct 10.0}}{{.Pct | printf "%.0f"}}{{else if ge .Pct 1.0}}{{.Pct | printf "%.01f"}}{{else}}{{.Pct | printf "%.02f"}}{{end}}%</td>
+                      <td style="width: 40%" {{if lt .Pct 5.0}}data-toggle="tooltip" title='{{.Count}}'{{end}}>
+                          <div class="progress-bar" role="progressbar" aria-valuenow="{{.Pct | printf "%.02f"}}" aria-valuemin="0" aria-valuemax="100" style="width: {{.Pct | printf "%.02f"}}%; height:20px" {{if ge .Pct 5.0}}data-toggle="tooltip" title='{{.Count}}'{{end}}></div>
+                      </td>
+                  </tr>
+                  {{end}}
+              </tbody>
+          </table>
+      </div>
+    {{end}}
+    </div>
+
+    <div class="row">
+        <div class="col-md-12">
+            <h4 id="features">Feature Group Usage</h4>
+            <p>
+                The following lists feature usage groups, which might include multiple occourances of a feature use per report.
+            </p>
+        </div>
+    </div>
+
+    <div class="row">
+    {{$i := counter}}
+    {{range $featureName := .featureOrder}}
+      {{$featureValues := index $.featureGroups $featureName }}
+      {{if $i.DrawTwoDivider}}
+        </div>
+        <div class="row">
+      {{end}}
+      {{ $i.Increment }}
+      <div class="col-md-6">
+          <table class="table table-striped">
+              <thead><tr>
+                  <th>{{$featureName}} Group Features</th><th colspan="2" class="text-center">Usage</th>
+              </tr></thead>
+              <tbody>
+                  {{range $featureValues}}
+                  {{$counts := .Counts}}
+                  <tr>
+                      <td style="width: 50%">
+                          <div data-toggle="tooltip" title='{{range $key, $value := .Counts}}{{$key}} ({{$value | proportion $counts | printf "%.02f"}}% - {{$value}})</br>{{end}}'>
+                              {{.Key}} ({{.Version}})
+                          </div>
+                      </td>
+                      <td style="width: 50%">
+                          <div class="progress" role="progressbar" style="width: 100%">
+                          {{$j := counter}}
+                          {{range $key, $value := .Counts}}
+                              {{with $valuePct := $value | proportion $counts}}
+                              <div class="progress-bar {{ $j.Current | progressBarClassByIndex }}" style='width: {{$valuePct | printf "%.02f"}}%' data-toggle="tooltip" title='{{$key}} ({{$valuePct | printf "%.02f"}}% - {{$value}})'>
+                                  {{if ge $valuePct 30.0}}{{$key}}{{end}}
+                              </div>
+                              {{end}}
+                              {{ $j.Increment }}
+                          {{end}}
+                          </div>
+                      </td>
+                  </tr>
+                  {{end}}
+              </tbody>
+          </table>
+      </div>
+    {{end}}
+    </div>
+     <div class="row">
+      <div class="col-md-12">
+        <h1 id="performance-charts">Historical Performance Data</h1>
+        <p>These charts are all the average of the corresponding metric, for the entire population of a given day.</p>
+
+        <h4 id="hash-performance">Hash Performance (MiB/s)</h4>
+        <div class="img-thumbnail" id="hashPerfChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+
+        <h4 id="memory-usage">Memory Usage (MiB)</h4>
+        <div class="img-thumbnail" id="memUsageChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+
+        <h4 id="total-files">Total Number of Files</h4>
+        <div class="img-thumbnail" id="totFilesChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+
+        <h4 id="total-size">Total Folder Size (GiB)</h4>
+        <div class="img-thumbnail" id="totMiBChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+
+        <h4 id="system-ram">System RAM Size (GiB)</h4>
+        <div class="img-thumbnail" id="memSizeChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
+      </div>
+    </div>
+  </div>
+  <hr>
+  <p>
+    This product includes GeoLite2 data created by MaxMind, available from
+    <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
+  </p>
+  <script type="text/javascript">
+    $('[data-toggle="tooltip"]').tooltip({html:true});
+  </script>
+</body>
+</html>

etc/linux-desktop/README.md

@@ -0,0 +1,12 @@
+# Desktop Entries
+
+This directory contains files to integrate Syncthing in your desktop environment (DE).
+Specifically this works for DEs that implement the [XDG Desktop Menu Specification][1], which
+is virtually every DE.  
+To add Syncthing to desktop menus for all users, copy the `.desktop` files to
+`/usr/local/share/applications` (root required). To add it for just your user, copy them to `~/.local/share/applications`.  
+To start Syncthing automatically, you have two options: Either you go to the autostart settings of your DE and choose Syncthing or you copy the `syncthing-start.desktop` file to `~/.config/autostart`.  
+For more information refer to the [ArchWiki page on Desktop entries][2]
+
+[1]: https://specifications.freedesktop.org/menu-spec/menu-spec-latest.html
+[2]: https://wiki.archlinux.org/index.php/Desktop_entries

etc/linux-desktop/syncthing-start.desktop

@@ -0,0 +1,10 @@
+[Desktop Entry]
+Name=Start Syncthing
+GenericName=File synchronization
+Comment=Starts the main syncthing process in the background.
+Exec=/usr/bin/syncthing -no-browser
+Icon=syncthing
+Terminal=false
+Type=Application
+Keywords=synchronization;daemon;
+Categories=Network;FileTransfer;P2P

etc/linux-desktop/syncthing-ui.desktop

@@ -0,0 +1,10 @@
+[Desktop Entry]
+Name=Syncthing Web UI
+GenericName=File synchronization UI
+Comment=Opens Syncthing's Web UI in the default browser (Syncthing must already be started).
+Exec=/usr/bin/syncthing -browser-only
+Icon=syncthing
+Terminal=false
+Type=Application
+Keywords=synchronization;interface;
+Categories=Network;FileTransfer;P2P

etc/linux-systemd/system/syncthing@.service

@@ -10,5 +10,12 @@ Restart=on-failure
 SuccessExitStatus=3 4
 RestartForceExitStatus=3 4
 
+# Hardening
+ProtectSystem=full
+PrivateTmp=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+
 [Install]
 WantedBy=multi-user.target

etc/linux-systemd/user/syncthing.service

@@ -8,5 +8,10 @@ Restart=on-failure
 SuccessExitStatus=3 4
 RestartForceExitStatus=3 4
 
+# Hardening
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+
 [Install]
 WantedBy=default.target

go.mod

@@ -0,0 +1,53 @@
+module github.com/syncthing/syncthing
+
+require (
+	github.com/AudriusButkevicius/go-nat-pmp v0.0.0-20160522074932-452c97607362
+	github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6
+	github.com/AudriusButkevicius/recli v0.0.5
+	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
+	github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e
+	github.com/calmh/xdr v1.1.0
+	github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d
+	github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67 // indirect
+	github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5
+	github.com/d4l3k/messagediff v1.2.1
+	github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
+	github.com/getsentry/raven-go v0.2.0
+	github.com/go-ole/go-ole v1.2.4 // indirect
+	github.com/gobwas/glob v0.2.3
+	github.com/gogo/protobuf v1.3.1
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6
+	github.com/golang/mock v1.3.1 // indirect
+	github.com/jackpal/gateway v1.0.5
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/lib/pq v1.2.0
+	github.com/lucas-clemente/quic-go v0.12.1
+	github.com/maruel/panicparse v1.3.0
+	github.com/mattn/go-isatty v0.0.10
+	github.com/minio/sha256-simd v0.1.1
+	github.com/onsi/ginkgo v1.9.0 // indirect
+	github.com/onsi/gomega v1.6.0 // indirect
+	github.com/oschwald/geoip2-golang v1.3.0
+	github.com/oschwald/maxminddb-golang v1.4.0 // indirect
+	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
+	github.com/pkg/errors v0.8.1
+	github.com/prometheus/client_golang v1.2.1
+	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563
+	github.com/sasha-s/go-deadlock v0.2.0
+	github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf
+	github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2
+	github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d
+	github.com/thejerf/suture v3.0.2+incompatible
+	github.com/urfave/cli v1.22.1
+	github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0
+	golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472
+	golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297
+	golang.org/x/text v0.3.2
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
+	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	gopkg.in/ldap.v2 v2.5.1
+)
+
+go 1.12

go.sum

@@ -0,0 +1,277 @@
+github.com/AudriusButkevicius/go-nat-pmp v0.0.0-20160522074932-452c97607362 h1:l4qGIzSY0WhdXdR74XMYAtfc0Ri/RJVM4p6x/E/+WkA=
+github.com/AudriusButkevicius/go-nat-pmp v0.0.0-20160522074932-452c97607362/go.mod h1:CEaBhA5lh1spxbPOELh5wNLKGsVQoahjUhVrJViVK8s=
+github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6 h1:Apvc4kyfdrOxG+F5dn8osz+45kwGJa6CySQn0tB38SU=
+github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6/go.mod h1:1N0EEx/irz4B1qV17wW82TFbjQrE7oX316Cki6eDY0Q=
+github.com/AudriusButkevicius/recli v0.0.5 h1:xUa55PvWTHBm17T6RvjElRO3y5tALpdceH86vhzQ5wg=
+github.com/AudriusButkevicius/recli v0.0.5/go.mod h1:Q2E26yc6RvWWEz/TJ/goUp6yXvipYdJI096hpoaqsNs=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e h1:2augTYh6E+XoNrrivZJBadpThP/dsvYKj0nzqfQ8tM4=
+github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
+github.com/calmh/xdr v1.1.0 h1:U/Dd4CXNLoo8EiQ4ulJUXkgO1/EyQLgDKLgpY1SOoJE=
+github.com/calmh/xdr v1.1.0/go.mod h1:E8sz2ByAdXC8MbANf1LCRYzedSnnc+/sXXJs/PVqoeg=
+github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d h1:As4937T5NVbJ/DmZT9z33pyLEprMd6CUSfhbmMY57Io=
+github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d/go.mod h1:3FK1bMar37f7jqVY7q/63k3OMX1c47pGCufzt3X0sYE=
+github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67 h1:8k9FLYBLKT+9v2HQJ/a95ZemmTx+/ltJcAiRhVushG8=
+github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
+github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
+github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
+github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
+github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
+github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5 h1:Wg96Dh0MLTanEaPO0OkGtUIaa2jOnShAIOVUIzRHUxo=
+github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U=
+github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI=
+github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
+github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
+github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
+github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc=
+github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lucas-clemente/quic-go v0.12.0 h1:TRbvZ6F++sofeGbh+Z2IIyIOhl8KyGnYuA06g2yrHdI=
+github.com/lucas-clemente/quic-go v0.12.0/go.mod h1:UXJJPE4RfFef/xPO5wQm0tITK8gNfqwTxjbE7s3Vb8s=
+github.com/lucas-clemente/quic-go v0.12.1 h1:BPITli+6KnKogtTxBk2aS4okr5dUHz2LtIDAP1b8UL4=
+github.com/lucas-clemente/quic-go v0.12.1/go.mod h1:UXJJPE4RfFef/xPO5wQm0tITK8gNfqwTxjbE7s3Vb8s=
+github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
+github.com/marten-seemann/qtls v0.3.2 h1:O7awy4bHEzSX/K3h+fZig3/Vo03s/RxlxgsAk9sYamI=
+github.com/marten-seemann/qtls v0.3.2/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
+github.com/maruel/panicparse v1.3.0 h1:1Ep/RaYoSL1r5rTILHQQbyzHG8T4UP5ZbQTYTo4bdDc=
+github.com/maruel/panicparse v1.3.0/go.mod h1:vszMjr5QQ4F5FSRfraldcIA/BCw5xrdLL+zEcU2nRBs=
+github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc=
+github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
+github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
+github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc=
+github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M=
+github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/oschwald/geoip2-golang v1.3.0 h1:D+Hsdos1NARPbzZ2aInUHZL+dApIzo8E0ErJVsWcku8=
+github.com/oschwald/geoip2-golang v1.3.0/go.mod h1:0LTTzix/Ao1uMvOhAV4iLU0Lz7eCrP94qZWBTDKf0iE=
+github.com/oschwald/maxminddb-golang v1.4.0 h1:5/rpmW41qrgSed4wK32rdznbkTSXHcraY2LOMJX4DMc=
+github.com/oschwald/maxminddb-golang v1.4.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
+github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ=
+github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8=
+github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI=
+github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo=
+github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
+github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.0.4 h1:w8DjqFMJDjuVwdZBQoOozr4MVWOnwF7RcL/7uxBjY78=
+github.com/prometheus/procfs v0.0.4/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
+github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
+github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y=
+github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
+github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf h1:c9SV5NzG4KOk448TUE7iqCmb4E4y79CZF4zDdc1Jx3Q=
+github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2 h1:6tuEEEpg+mxM82E0YingzoXzXXISYR/o/7I9n573LWI=
+github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2/go.mod h1:Sn4ChoS7e4FxjCN1XHPVBT43AgnRLbuaB8pEc1Zcdjg=
+github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
+github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
+github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs=
+github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
+github.com/thejerf/suture v3.0.2+incompatible h1:GtMydYcnK4zBJ0KL6Lx9vLzl6Oozb65wh252FTBxrvM=
+github.com/thejerf/suture v3.0.2+incompatible/go.mod h1:ibKwrVj+Uzf3XZdAiNWUouPaAbSoemxOHLmJmwheEMc=
+github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
+github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0 h1:okhMind4q9H1OxF44gNegWkiP4H/gsTFLalHFa4OOUI=
+github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0/go.mod h1:TTbGUfE+cXXceWtbTHq6lqcTvYPBKLNejBEbnUsQJtU=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM=
+golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e h1:ZytStCyV048ZqDsWHiYDdoI2Vd4msMcrDECFxS+tL9c=
+golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
+gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/ldap.v2 v2.5.1 h1:wiu0okdNfjlBzg6UWvd1Hn8Y+Ux17/u/4nlk4CQr6tU=
+gopkg.in/ldap.v2 v2.5.1/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

gui/black/assets/css/theme.css

@@ -24,6 +24,27 @@ a:hover,a:focus,a.focus{
     border-width: 2px !important;
 }
 
+
+.nav-tabs > li.active > a,
+.nav-tabs > li.active > a:hover,
+.nav-tabs > li.active > a:focus {
+  color: #3498db !important;
+  background-color: #222222 !important;
+  border: 1px solid #222222 !important;
+  border-bottom-color: transparent !important;
+  cursor: default;
+}
+
+.nav-tabs{
+    border-bottom: 1px solid #333;
+}
+
+.nav-tabs > li > a:hover,
+.nav-tabs > li > a:focus {
+  background-color: #222222 !important;
+  border: none !important;
+}
+
 .navbar-text, .dropdown>a, .dropdown-menu>li>a, .hidden-xs>a, .navbar-link {
     color: #aaa !important;
 }

gui/dark/assets/css/theme.css

@@ -28,6 +28,27 @@ a:hover,a:focus,a.focus{
 
 }
 
+.nav-tabs > li.active > a,
+.nav-tabs > li.active > a:hover,
+.nav-tabs > li.active > a:focus {
+  color: #3498db !important;
+  background-color: #424242 !important;
+  border: 1px solid #424242 !important;
+  border-bottom-color: transparent !important;
+  cursor: default;
+}
+
+.nav-tabs{
+    border-bottom: 1px solid #333;
+}
+
+.nav-tabs > li > a:hover,
+.nav-tabs > li > a:focus {
+  background-color: #424242 !important;
+  border: none !important;
+}
+
+
 .navbar-text, .dropdown>a, .dropdown-menu>li>a, .hidden-xs>a, .navbar-link {
     color: #aaa !important;
 }