cmd/syncthing: Correct auto upgrade criteria (fixes #6701) (#6702)

We can't just drop the snap because it's in use elsewhere. This should
be equally functional.

.codecov.yml

@@ -0,0 +1,5 @@
+coverage:
+  range: "40...100"
+
+ignore:
+  - "**.pb.go"

.github/ISSUE_TEMPLATE.md

@@ -1,42 +0,0 @@
-### DO NOT REPORT SECURITY ISSUES IN THIS ISSUE TRACKER
-
-Instead, contact security@syncthing.net directly - see
-https://syncthing.net/security.html for more information.
-
-### DO NOT POST SUPPORT REQUESTS OR GENERAL QUESTIONS IN THIS ISSUE TRACKER
-
-Please use the forum at https://forum.syncthing.net/ where a large number of
-helpful people hang out. This issue tracker is for reporting bugs or feature
-requests directly to the developers. Worst case you might get a short
-"that's a bug, please report it on GitHub" response on the forum, in which
-case we thank you for your patience and following our advice. :)
-
-### Please use the correct issue tracker
-
-If your problem relates to a Syncthing wrapper or [sub-project](https://github.com/syncthing) such as [Syncthing for Android](https://github.com/syncthing/syncthing-android/issues), [SyncTrayzor](https://github.com/canton7/synctrayzor) or the [documentation](https://github.com/syncthing/docs/issues), please use their respective issue trackers.
-
-### Does your log mention database corruption?
-
-If your Syncthing log reports panics because of database corruption it is most likely a fault with your system's storage or memory. Affected log entries will contain lines starting with `panic: leveldb`. You will need to delete the index database to clear this, by running `syncthing -reset-database`.
-
-### Please do post actual bug reports and feature requests.
-
-If your issue is a bug report, replace this boilerplate with a description
-of the problem, being sure to include at least:
-
- - what happened,
- - what you expected to happen instead, and
- - any steps to reproduce the problem.
-
-Also fill out the version information below and add log output or
-screenshots as appropriate.
-
-If your issue is a feature request, simply replace this template text in
-its entirety.
-
-### Version Information
-
-Syncthing Version: v0.x.y
-OS Version: Windows 7 / Ubuntu 14.04 / ...
-Browser Version: (if applicable, for GUI issues)
-

.github/ISSUE_TEMPLATE/01-feature.md

@@ -0,0 +1,13 @@
+---
+name: Feature request
+about: If you're just not sure how to do something, see "ask a question".
+labels: enhancement, needs-triage
+---
+
+### Include required information
+
+Please be sure to include at least:
+
+ - what problem your new feature would solve
+ - how or why you think it is generally useful (i.e., not just for you)
+ - what alternatives or workarounds you considered

.github/ISSUE_TEMPLATE/02-bug.md

@@ -0,0 +1,23 @@
+---
+name: Bug report
+about: If you're actually looking for support, see "ask a question".
+labels: bug, needs-triage
+---
+
+### Does your log mention database corruption?
+
+If your Syncthing log reports panics because of database corruption it is
+most likely a fault with your system's storage or memory. Affected log
+entries will contain lines starting with `panic: leveldb`. You will need to
+delete the index database to clear this, by running `syncthing
+-reset-database`.
+
+### Include required information
+
+Please be sure to include at least:
+
+ - which version of Syncthing and what operating system you are using
+ - browser and version, if applicable
+ - what happened,
+ - what you expected to happen instead, and
+ - any steps to reproduce the problem.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a question
+    url: https://forum.syncthing.net/
+    about: Ask questions, get support, and discuss with other community members.
+  - name: Android issues
+    url: https://github.com/syncthing/syncthing-android/issues/
+    about: The Android app has its own issue tracker.

.github/SECURITY.md

@@ -0,0 +1,10 @@
+## Reporting a Vulnerability
+
+If you believe that you've found a Syncthing-related security vulnerability,
+please report it by sending email to the address security@syncthing.net. The
+[PGP key for security@syncthing.net
+(B683AD7B76CAB013)](https://syncthing.net/security-key.txt) can be used to
+send encrypted mail or to verify responses received from that address.
+
+You can read more about Syncthing security at
+https://syncthing.net/security/.

.gitignore

@@ -15,11 +15,5 @@ coverage.xml
 syncthing.sig
 RELEASE
 deb
-lib/auto/gui.files.go
-snapcraft.yaml
-prime/
-snap/
-parts/
-stage/
-*.snap
 *.bz2
+/repos

.golangci.yml

@@ -0,0 +1,26 @@
+linters-settings:
+  maligned:
+    suggest-new: true
+
+linters:
+  enable-all: true
+  disable:
+    - goimports
+    - depguard
+    - lll
+    - gochecknoinits
+    - gochecknoglobals
+    - gofmt
+    - scopelint
+    - gocyclo
+    - funlen
+    - wsl
+    - gocognit
+    - godox
+
+service:
+  golangci-lint-version: 1.21.x
+  prepare:
+    - rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
+    - GO111MODULE=on go mod vendor
+    - go run build.go assets

AUTHORS

@@ -16,20 +16,30 @@
 Aaron Bieber (qbit) <qbit@deftly.net>
 Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com> <adam@proactiveservices.co.uk>
 Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
+Alan Pope <alan@popey.com>
+Alberto Donato <albertodonato@users.noreply.github.com>
 Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
+Alex Xu <alex.hello71@gmail.com>
 Alexander Graf (alex2108) <register-github@alex-graf.de>
 Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
+Aman Gupta <aman@tmm1.net>
 Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
 andresvia <andres.via@gmail.com>
 Andrew Dunham (andrew-d) <andrew@du.nham.ca>
 Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
 Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
+André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
 andyleap <andyleap@gmail.com>
+Anjan Momi <anjan@momi.ca>
 Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
 Antony Male (canton7) <antony.male@gmail.com>
 Aranjedeath <Aranjedeath@users.noreply.github.com>
+Arkadiusz Tymiński <gevleeog@gmail.com>
 Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
-Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com>
+Artur Zubilewicz <AkaZecik@users.noreply.github.com>
+Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
+Aurélien Rainone <476650+arl@users.noreply.github.com>
+BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
 Bart De Vries (mogwa1) <devriesb@gmail.com>
 Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
 Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
@@ -37,21 +47,28 @@ Ben Shepherd (benshep) <bjashepherd@gmail.com>
 Ben Sidhom (bsidhom) <bsidhom@gmail.com>
 Benedikt Heine (bebehei) <bebe@bebehei.de>
 Benedikt Morbach <benedikt.morbach@googlemail.com>
+Benno Fünfstück <benno.fuenfstueck@gmail.com>
 Benny Ng (tpng) <benny.tpng@gmail.com>
+boomsquared <54829195+boomsquared@users.noreply.github.com>
+Boqin Qin <bobbqqin@bupt.edu.cn>
 Boris Rybalkin <ribalkin@gmail.com>
 Brandon Philips (philips) <brandon@ifup.org>
 Brendan Long (brendanlong) <self@brendanlong.com>
 Brian R. Becker (brbecker) <brbecker@gmail.com>
 Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
-Carsten Hagemann (Moter8) <moter8@gmail.com>
+Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
 Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
 Cedric Staniewski (xduugu) <cedric@gmx.ca>
+chenrui <rui@meetup.com>
 Chris Howie (cdhowie) <me@chrishowie.com>
 Chris Joel (cdata) <chris@scriptolo.gy>
 Chris Tonkinson <chris@masterbran.ch>
 chucic <chucic@seznam.cz>
 Colin Kennedy (moshen) <moshen.colin@gmail.com>
+Cromefire_ <tim.l@nghorst.net> <26320625+cromefire@users.noreply.github.com>
+Cyprien Devillez <cypx@users.noreply.github.com>
 Dale Visser <dale.visser@live.com>
+Dan <benda.daniel@gmail.com>
 Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
 Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
 Daniel Martí (mvdan) <mvdan@mvdan.cc>
@@ -59,27 +76,39 @@ Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
 David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
 Denis A. (dva) <denisva@gmail.com>
 Dennis Wilson (snnd) <dw@risu.io>
+dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com> <27856297+dependabot-preview[bot]@users.noreply.github.com>
+dependabot[bot] <dependabot[bot]@users.noreply.github.com>
 derekriemer <derek.riemer@colorado.edu>
+desbma <desbma@users.noreply.github.com>
 Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
+Domenic Horner <domenic@tgxn.net>
 Dominik Heidler (asdil12) <dominik@heidler.eu>
 Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
 Elliot Huffman <thelich2@gmail.com>
 Emil Hessman (ceh) <emil@hessman.se>
 Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
+Evgeny Kuznetsov <evgeny@kuznetsov.md>
 Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
 Felix Ableitner (Nutomic) <me@nutomic.com>
 Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
 Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
 Frank Isemann (fti7) <frank@isemann.name>
+georgespatton <georgespatton@users.noreply.github.com>
+ghjklw <malo@jaffre.info>
 Gilli Sigurdsson (gillisig) <gilli@vx.is>
 Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
+greatroar <61184462+greatroar@users.noreply.github.com>
 Han Boetes <han@boetes.org>
 Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
 Heiko Zuerker (Smiley73) <heiko@zuerker.org>
+Hugo Locurcio <hugo.locurcio@hugo.pro>
 Iain Barnett <iainspeed@gmail.com>
 Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
+Ilya Brin <464157+ilyabrin@users.noreply.github.com>
+Iskander Sharipov (Alex) <quasilyte@gmail.com>
 Jaakko Hannikainen (jgke) <jgke@jgke.fi>
 Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
+Jacob <jyundt@gmail.com>
 Jake Peterson (acogdev) <jake@acogdev.com>
 Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
 James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
@@ -87,19 +116,24 @@ janost <janost@tuta.io>
 Jaroslav Malec (dzarda) <dzardacz@gmail.com>
 jaseg <githubaccount@jaseg.net>
 Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
+jelle van der Waa <jelle@vdwaa.nl>
 Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
 Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
 Jochen Voss (seehuhn) <voss@seehuhn.de>
 Johan Andersson <j@i19.se>
 Johan Vromans (sciurius) <jvromans@squirrel.nl>
 John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
+Jonas Thelemann <e-mail@jonas-thelemann.de>
 Jonathan Cross <jcross@gmail.com>
 Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
+Jörg Thalheim <Mic92@users.noreply.github.com>
+Kalle Laine <pahakalle@protonmail.com>
 Karol Różycki (krozycki) <rozycki.karol@gmail.com>
 Keith Turner <kturner@apache.org>
 Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
 Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
 Kevin Allen (ironmig) <kma1660@gmail.com>
+Kevin Bushiri (keevBush) <keevbush@gmail.com> <36192217+keevBush@users.noreply.github.com>
 Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
 klemens <ka7@github.com>
 Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
@@ -110,35 +144,54 @@ Leo Arias (elopio) <yo@elopio.net>
 Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
 Lode Hoste (Zillode) <zillode@zillode.be>
 Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
+Lukas Lihotzki <lukas@lihotzki.de>
 Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
 Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
 Marc Pujol (kilburn) <kilburn@la3.org>
 Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
 marco-m <marco.molteni@laposte.net>
+Marcus Legendre <marcus.legendre@gmail.com>
+Mario Majila <mariustshipichik@gmail.com>
 Mark Pulford (mpx) <mark@kyne.com.au>
 Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
+Mateusz Ż <thedead4fun@live.com>
 Matic Potočnik <hairyfotr@gmail.com>
 Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
+Matt Robenolt <matt@ydekproductions.com>
 Matteo Ruina <matteo.ruina@gmail.com>
+Maurizio Tomasi <ziotom78@gmail.com>
 Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
 MaximAL <almaximal@ya.ru>
 Maxime Thirouin <m@moox.io>
 Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
 Michael Ploujnikov (plouj) <ploujj@gmail.com>
+Michael Rienstra <mrienstra@gmail.com>
 Michael Tilli (pyfisch) <pyfisch@gmail.com>
 Mike Boone <mike@boonedocks.net>
 MikeLund <MikeLund@users.noreply.github.com>
+MikolajTwarog <43782609+MikolajTwarog@users.noreply.github.com>
+Mingxuan Lin <gdlmx@users.noreply.github.com>
+mv1005 <49659413+mv1005@users.noreply.github.com>
 Nate Morrison (nrm21) <natemorrison@gmail.com>
 Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
+Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
 Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
+Nicolas Perraut <n.perraut@gmail.com>
 Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
 Nils Jakobi (thunderstorm99) <jakobi.nils@gmail.com>
+Nitroretro <43112364+Nitroretro@users.noreply.github.com>
 NoLooseEnds <jon.koslung@gmail.com>
+Oliver Freyermuth <o.freyermuth@googlemail.com>
+otbutz <tbutz@optitool.de>
+Otiel <Otiel@users.noreply.github.com>
 Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
+Pablo <pbaeyens31+github@gmail.com>
 Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
+Paul Brit <paulbrit44@gmail.com>
 Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
 Paweł Rozlach <vespian@users.noreply.github.com>
 perewa <cavalcante.ten@gmail.com>
+Peter Badida <KeyWeeUsr@users.noreply.github.com>
 Peter Dave Hello <hsu@peterdavehello.org>
 Peter Hoeg (peterhoeg) <peter@speartail.com>
 Peter Marquardt (wwwutz) <wwwutz@gmail.com> <wwwutz@googlemail.com>
@@ -150,17 +203,20 @@ Piotr Bejda (piobpl) <piotrb10@gmail.com>
 Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
 Richard Hartmann <RichiH@users.noreply.github.com>
 Robert Carosi (nov1n) <robert@carosi.nl>
+Robin Schoonover <robin@cornhooves.org>
 Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
 Ross Smith II (rasa) <ross@smithii.com>
 rubenbe <github-com-00ff86@vandamme.email>
+Ruslan Yevdokymov <38809160+ruslanye@users.noreply.github.com>
 Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
 Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
 Scott Klupfel (kluppy) <kluppy@going2blue.com>
 Sergey Mishin (ralder) <ralder@yandex.ru>
 Simon Frei (imsodin) <freisim93@gmail.com>
+Simon Mwepu <simonmwepu@gmail.com>
 Sly_tom_cat <slytomcat@mail.ru>
 Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
-Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
+Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
 Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
 Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
 Thomas Hipp <thomashipp@gmail.com>
@@ -168,10 +224,12 @@ Tim Abell (timabell) <tim@timwise.co.uk>
 Tim Howes (timhowes) <timhowes@berkeley.edu>
 Tobias Nygren (tnn2) <tnn@nygren.pp.se>
 Tobias Tom (tobiastom) <t.tom@succont.de>
-Tomas Cerveny (kozec) <kozec@kozec.com>
+Tom Jakubowski <tom@crystae.net>
+Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com>
 Tommy Thorn <tommy-github-email@thorn.ws>
 Tully Robinson (tojrobinson) <tully@tojr.org>
 Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
+Tyler Kropp <kropptyler@gmail.com>
 Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
 Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
 Victor Buinsky (buinsky) <vix_booja@tut.by>

Dockerfile

@@ -1,6 +1,7 @@
-FROM golang:1.11 AS builder
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
 
-WORKDIR /go/src/github.com/syncthing/syncthing
+WORKDIR /src
 COPY . .
 
 ENV CGO_ENABLED=0
@@ -16,17 +17,10 @@ VOLUME ["/var/syncthing"]
 
 RUN apk add --no-cache ca-certificates su-exec
 
-COPY --from=builder /go/src/github.com/syncthing/syncthing/syncthing /bin/syncthing
+COPY --from=builder /src/syncthing /bin/syncthing
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
 
-ENV PUID=1000 PGID=1000
+ENV PUID=1000 PGID=1000 HOME=/var/syncthing
 
-HEALTHCHECK --interval=1m --timeout=10s \
+ENV STGUIADDRESS=0.0.0.0:8384
-  CMD nc -z localhost 8384 || exit 1
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]
-
-ENTRYPOINT \
-  chown "${PUID}:${PGID}" /var/syncthing \
-  && su-exec "${PUID}:${PGID}" \
-     env HOME=/var/syncthing \
-     /bin/syncthing \
-       -home /var/syncthing/config \
-       -gui-address 0.0.0.0:8384

Dockerfile.builder

@@ -0,0 +1,9 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION
+
+# FPM to build Debian packages
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	locales rubygems ruby-dev build-essential git \
+	&& apt-get clean \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& gem install --no-ri --no-rdoc fpm

Dockerfile.stdiscosrv

@@ -0,0 +1,29 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f stdiscosrv && go run build.go -no-upgrade build stdiscosrv
+
+FROM alpine
+
+EXPOSE 19200 8443
+
+VOLUME ["/var/stdiscosrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/stdiscosrv /bin/stdiscosrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/stdiscosrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 8443 || exit 1
+
+WORKDIR /var/stdiscosrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/stdiscosrv"]

Dockerfile.strelaysrv

@@ -0,0 +1,29 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f strelaysrv && go run build.go -no-upgrade build strelaysrv
+
+FROM alpine
+
+EXPOSE 22067 22070
+
+VOLUME ["/var/strelaysrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/strelaysrv /bin/strelaysrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/strelaysrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 22067 || exit 1
+
+WORKDIR /var/strelaysrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaysrv"]

README-Docker.md

@@ -1,39 +1,34 @@
 # Docker Container for Syncthing
 
 Use the Dockerfile in this repo, or pull the `syncthing/syncthing` image
-from Docker Hub. Use volumes to have the synchronized files available on the
+from Docker Hub.
-host.
 
-The exposed volumes are by default:
+Use the `/var/syncthing` volume to have the synchronized files available on the
-
+host. You can add more folders and map them as you prefer.
-    /var/syncthing/config   - the configuration and index directory into the Container
-    /var/syncthing          - the default sync folder into the Container
-
-You can add more folders and map them as you prefer.
 
 Note that Syncthing runs as UID 1000 and GID 1000 by default. These may be
 altered with the ``PUID`` and ``PGID`` environment variables.
 
-Example usage:
+## Example Usage
 
 ```
 $ docker pull syncthing/syncthing
 $ docker run -p 8384:8384 -p 22000:22000 \
-    -v /wherever/st-cfg:/var/syncthing/config \
     -v /wherever/st-sync:/var/syncthing \
     syncthing/syncthing:latest
 ```
 
-Note that local device discovery will not work with the above command resulting
+## Discovery
-in poor local transfer rates if local device addresses are not manually
+
-configured.
+Note that local device discovery will not work with the above command,
+resulting in poor local transfer rates if local device addresses are not
+manually configured.
 
 To allow local discovery, the docker host network can be used instead:
 
 ```
 $ docker pull syncthing/syncthing
 $ docker run --network=host \
-    -v /wherever/st-cfg:/var/syncthing/config \
     -v /wherever/st-sync:/var/syncthing \
     syncthing/syncthing:latest
 ```
@@ -41,3 +36,24 @@ $ docker run --network=host \
 Be aware that syncthing alone is now in control of what interfaces and ports it
 listens on. You can edit the syncthing configuration to change the defaults if
 there are conflicts.
+
+## GUI Security
+
+By default Syncthing inside the Docker image listens on 0.0.0.0:8384 to
+allow GUI connections via the Docker proxy. This is set by the
+`STGUIADDRESS` environment variable in the Dockerfile, as it differs from
+what Syncthing would otherwise use by default. This means you should set up
+authentication in the GUI, like for any other externally reachable Syncthing
+instance. If you do not require the GUI, or you use host networking, you can
+unset the `STGUIADDRESS` variable to have Syncthing fall back to listening
+on 127.0.0.1:
+
+```
+$ docker pull syncthing/syncthing
+$ docker run -e STGUIADDRESS= \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+With the environment variable unset Syncthing will follow what is set in the
+configuration file / GUI settings dialog.

README.md

@@ -2,7 +2,6 @@
 
 ---
 
-[![Latest Downloads](https://img.shields.io/badge/latest-downloads-brightgreen.svg?style=flat-square)](https://build.syncthing.net/latest/)
 [![Latest Linux & Cross Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildLinuxCross.svg?style=flat-square&label=linux+%26+cross+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildLinuxCross&guest=1)
 [![Latest Windows Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildWindows.svg?style=flat-square&label=windows+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildWindows&guest=1)
 [![Latest Mac Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildMac.svg?style=flat-square&label=mac+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildMac&guest=1)
@@ -63,6 +62,10 @@ There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory][3]. There are also several [GUI
 implementations][11] for Windows, Mac and Linux.
 
+## Docker
+
+To run Syncthing in Docker, see [the Docker README][16].
+
 ## Vote on features/bugs
 
 We'd like to encourage you to [vote][12] on issues that matter to you.
@@ -111,4 +114,5 @@ All code is licensed under the [MPLv2 License][7].
 [13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
 [14]: assets/logo-text-128.png
 [15]: https://syncthing.net/
+[16]: https://github.com/syncthing/syncthing/blob/master/README-Docker.md
 

build.go

@@ -15,6 +15,7 @@ import (
 	"compress/flate"
 	"compress/gzip"
 	"crypto/sha256"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -29,7 +30,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"text/template"
 	"time"
 )
 
@@ -39,14 +39,15 @@ var (
 	goos          string
 	noupgrade     bool
 	version       string
-	goVersion     float64
+	goCmd         string
 	race          bool
 	debug         = os.Getenv("BUILDDEBUG") != ""
-	noBuildGopath bool
 	extraTags     string
 	installSuffix string
 	pkgdir        string
+	cc            string
 	debugBinary   bool
+	coverage      bool
 	timeout       = "120s"
 )
 
@@ -54,11 +55,13 @@ type target struct {
 	name              string
 	debname           string
 	debdeps           []string
+	debpre            string
 	debpost           string
 	description       string
-	buildPkg          string
+	buildPkgs         []string
 	binaryName        string
 	archiveFiles      []archiveFile
+	systemdServices   []string
 	installationFiles []archiveFile
 	tags              []string
 }
@@ -72,9 +75,8 @@ type archiveFile struct {
 var targets = map[string]target{
 	"all": {
 		// Only valid for the "build" and "install" commands as it lacks all
-		// the archive creation stuff.
+		// the archive creation stuff. buildPkgs gets filled out in init()
-		buildPkg: "github.com/syncthing/syncthing/cmd/...",
+		tags: []string{"purego"},
-		tags:     []string{"purego"},
 	},
 	"syncthing": {
 		// The default target for "build", "install", "tar", "zip", "deb", etc.
@@ -83,7 +85,7 @@ var targets = map[string]target{
 		debdeps:     []string{"libc6", "procps"},
 		debpost:     "script/post-upgrade",
 		description: "Open Source Continuous File Synchronization",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/syncthing",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/syncthing"},
 		binaryName:  "syncthing", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -111,14 +113,23 @@ var targets = map[string]target{
 			{src: "etc/linux-systemd/system/syncthing-resume.service", dst: "deb/lib/systemd/system/syncthing-resume.service", perm: 0644},
 			{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0644},
 			{src: "etc/firewall-ufw/syncthing", dst: "deb/etc/ufw/applications.d/syncthing", perm: 0644},
+			{src: "etc/linux-desktop/syncthing-start.desktop", dst: "deb/usr/share/applications/syncthing-start.desktop", perm: 0644},
+			{src: "etc/linux-desktop/syncthing-ui.desktop", dst: "deb/usr/share/applications/syncthing-ui.desktop", perm: 0644},
+			{src: "assets/logo-32.png", dst: "deb/usr/share/icons/hicolor/32x32/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-64.png", dst: "deb/usr/share/icons/hicolor/64x64/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-128.png", dst: "deb/usr/share/icons/hicolor/128x128/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-256.png", dst: "deb/usr/share/icons/hicolor/256x256/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-512.png", dst: "deb/usr/share/icons/hicolor/512x512/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-only.svg", dst: "deb/usr/share/icons/hicolor/scalable/apps/syncthing.svg", perm: 0644},
 		},
 	},
 	"stdiscosrv": {
 		name:        "stdiscosrv",
 		debname:     "syncthing-discosrv",
 		debdeps:     []string{"libc6"},
+		debpre:      "cmd/stdiscosrv/scripts/preinst",
 		description: "Syncthing Discovery Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/stdiscosrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/stdiscosrv"},
 		binaryName:  "stdiscosrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -126,12 +137,17 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
+		systemdServices: []string{
+			"cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service",
+		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
 			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0644},
 			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0644},
 			{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-discosrv", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv", dst: "deb/etc/ufw/applications.d/stdiscosrv", perm: 0644},
 		},
 		tags: []string{"purego"},
 	},
@@ -139,8 +155,9 @@ var targets = map[string]target{
 		name:        "strelaysrv",
 		debname:     "syncthing-relaysrv",
 		debdeps:     []string{"libc6"},
+		debpre:      "cmd/strelaysrv/scripts/preinst",
 		description: "Syncthing Relay Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/strelaysrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaysrv"},
 		binaryName:  "strelaysrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -149,6 +166,9 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
+		systemdServices: []string{
+			"cmd/strelaysrv/etc/linux-systemd/strelaysrv.service",
+		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
 			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0644},
@@ -156,6 +176,8 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0644},
 			{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0644},
+			{src: "cmd/strelaysrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-relaysrv", perm: 0644},
+			{src: "cmd/strelaysrv/etc/firewall-ufw/strelaysrv", dst: "deb/etc/ufw/applications.d/strelaysrv", perm: 0644},
 		},
 	},
 	"strelaypoolsrv": {
@@ -163,7 +185,7 @@ var targets = map[string]target{
 		debname:     "syncthing-relaypoolsrv",
 		debdeps:     []string{"libc6"},
 		description: "Syncthing Relay Pool Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/strelaypoolsrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaypoolsrv"},
 		binaryName:  "strelaypoolsrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -180,7 +202,31 @@ var targets = map[string]target{
 	},
 }
 
+// These are repos we need to clone to run "go generate"
+
+type dependencyRepo struct {
+	path   string
+	repo   string
+	commit string
+}
+
+var dependencyRepos = []dependencyRepo{
+	{path: "xdr", repo: "https://github.com/calmh/xdr.git", commit: "08e072f9cb16"},
+}
+
 func init() {
+	all := targets["all"]
+	pkgs, _ := filepath.Glob("cmd/*")
+	for _, pkg := range pkgs {
+		pkg = filepath.Base(pkg)
+		if strings.HasPrefix(pkg, ".") {
+			// ignore dotfiles
+			continue
+		}
+		all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/cmd/%s", pkg))
+	}
+	targets["all"] = all
+
 	// The "syncthing" target includes a few more files found in the "etc"
 	// and "extra" dirs.
 	syncthingPkg := targets["syncthing"]
@@ -208,39 +254,6 @@ func main() {
 		}()
 	}
 
-	if gopath := gopath(); gopath == "" {
-		gopath, err := temporaryBuildDir()
-		if err != nil {
-			log.Fatal(err)
-		}
-		os.Setenv("GOPATH", gopath)
-		log.Println("GOPATH is", gopath)
-		if !noBuildGopath {
-			if err := buildGOPATH(gopath); err != nil {
-				log.Fatal(err)
-			}
-			lazyRebuildAssets()
-		}
-	} else {
-		inside := false
-		wd, _ := os.Getwd()
-		wd, _ = filepath.EvalSymlinks(wd)
-		for _, p := range filepath.SplitList(gopath) {
-			p, _ = filepath.EvalSymlinks(p)
-			if filepath.Join(p, "src/github.com/syncthing/syncthing") == wd {
-				inside = true
-				break
-			}
-		}
-		if !inside {
-			fmt.Println("You seem to have GOPATH set but the Syncthing source not placed correctly within it, which may cause problems.")
-		}
-	}
-
-	// Set path to $GOPATH/bin:$PATH so that we can for sure find tools we
-	// might have installed during "build.go setup".
-	os.Setenv("PATH", fmt.Sprintf("%s%cbin%c%s", os.Getenv("GOPATH"), os.PathSeparator, os.PathListSeparator, os.Getenv("PATH")))
-
 	// Invoking build.go with no parameters at all builds everything (incrementally),
 	// which is what you want for maximum error checking during development.
 	if flag.NArg() == 0 {
@@ -264,9 +277,6 @@ func main() {
 
 func runCommand(cmd string, target target) {
 	switch cmd {
-	case "setup":
-		setup()
-
 	case "install":
 		var tags []string
 		if noupgrade {
@@ -311,12 +321,6 @@ func runCommand(cmd string, target target) {
 	case "deb":
 		buildDeb(target)
 
-	case "snap":
-		buildSnap(target)
-
-	case "clean":
-		clean()
-
 	case "vet":
 		metalintShort()
 
@@ -329,13 +333,6 @@ func runCommand(cmd string, target target) {
 	case "version":
 		fmt.Println(getVersion())
 
-	case "gopath":
-		gopath, err := temporaryBuildDir()
-		if err != nil {
-			log.Fatal(err)
-		}
-		fmt.Println(gopath)
-
 	default:
 		log.Fatalf("Unknown command %q", cmd)
 	}
@@ -344,64 +341,41 @@ func runCommand(cmd string, target target) {
 func parseFlags() {
 	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
 	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
+	flag.StringVar(&goCmd, "gocmd", "go", "Specify `go` command")
 	flag.BoolVar(&noupgrade, "no-upgrade", noupgrade, "Disable upgrade functionality")
 	flag.StringVar(&version, "version", getVersion(), "Set compiled in version string")
 	flag.BoolVar(&race, "race", race, "Use race detector")
-	flag.BoolVar(&noBuildGopath, "no-build-gopath", noBuildGopath, "Don't build GOPATH, assume it's OK")
 	flag.StringVar(&extraTags, "tags", extraTags, "Extra tags, space separated")
 	flag.StringVar(&installSuffix, "installsuffix", installSuffix, "Install suffix, optional")
 	flag.StringVar(&pkgdir, "pkgdir", "", "Set -pkgdir parameter for `go build`")
+	flag.StringVar(&cc, "cc", os.Getenv("CC"), "Set CC environment variable for `go build`")
 	flag.BoolVar(&debugBinary, "debug-binary", debugBinary, "Create unoptimized binary to use with delve, set -gcflags='-N -l' and omit -ldflags")
+	flag.BoolVar(&coverage, "coverage", coverage, "Write coverage profile of tests to coverage.txt")
 	flag.Parse()
 }
 
-func setup() {
-	packages := []string{
-		"github.com/alecthomas/gometalinter",
-		"github.com/AlekSi/gocov-xml",
-		"github.com/axw/gocov/gocov",
-		"github.com/FiloSottile/gvt",
-		"github.com/golang/lint/golint",
-		"github.com/gordonklaus/ineffassign",
-		"github.com/mdempsky/unconvert",
-		"github.com/mitchellh/go-wordwrap",
-		"github.com/opennota/check/cmd/...",
-		"github.com/tsenart/deadcode",
-		"golang.org/x/net/html",
-		"golang.org/x/tools/cmd/cover",
-		"honnef.co/go/tools/cmd/gosimple",
-		"honnef.co/go/tools/cmd/staticcheck",
-		"honnef.co/go/tools/cmd/unused",
-		"github.com/josephspurrier/goversioninfo",
-	}
-	for _, pkg := range packages {
-		fmt.Println(pkg)
-		runPrint("go", "get", "-u", pkg)
-	}
-
-	runPrint("go", "install", "-v", "github.com/syncthing/syncthing/vendor/github.com/gogo/protobuf/protoc-gen-gogofast")
-}
-
 func test(pkgs ...string) {
 	lazyRebuildAssets()
 
-	useRace := runtime.GOARCH == "amd64"
+	args := []string{"test", "-short", "-timeout", timeout, "-tags", "purego"}
-	switch runtime.GOOS {
+
-	case "darwin", "linux", "freebsd", "windows":
+	if runtime.GOARCH == "amd64" {
-	default:
+		switch runtime.GOOS {
-		useRace = false
+		case "darwin", "linux", "freebsd": // , "windows": # See https://github.com/golang/go/issues/27089
+			args = append(args, "-race")
+		}
 	}
 
-	if useRace {
+	if coverage {
-		runPrint("go", append([]string{"test", "-short", "-race", "-timeout", timeout, "-tags", "purego"}, pkgs...)...)
+		args = append(args, "-covermode", "atomic", "-coverprofile", "coverage.txt", "-coverpkg", strings.Join(pkgs, ","))
-	} else {
-		runPrint("go", append([]string{"test", "-short", "-timeout", timeout, "-tags", "purego"}, pkgs...)...)
 	}
+
+	runPrint(goCmd, append(args, pkgs...)...)
 }
 
 func bench(pkgs ...string) {
 	lazyRebuildAssets()
-	runPrint("go", append([]string{"test", "-run", "NONE", "-bench", "."}, pkgs...)...)
+	runPrint(goCmd, append([]string{"test", "-run", "NONE", "-bench", "."}, pkgs...)...)
 }
 
 func install(target target, tags []string) {
@@ -415,11 +389,9 @@ func install(target target, tags []string) {
 	}
 	os.Setenv("GOBIN", filepath.Join(cwd, "bin"))
 
-	args := []string{"install", "-v"}
-	args = appendParameters(args, tags, target)
-
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
+	os.Setenv("CC", cc)
 
 	// On Windows generate a special file which the Go compiler will
 	// automatically use when generating Windows binaries to set things like
@@ -432,21 +404,20 @@ func install(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	runPrint("go", args...)
+	args := []string{"install", "-v"}
+	args = appendParameters(args, tags, target.buildPkgs...)
+	runPrint(goCmd, args...)
 }
 
 func build(target target, tags []string) {
 	lazyRebuildAssets()
-
 	tags = append(target.tags, tags...)
 
 	rmr(target.BinaryName())
 
-	args := []string{"build", "-i", "-v"}
-	args = appendParameters(args, tags, target)
-
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
+	os.Setenv("CC", cc)
 
 	// On Windows generate a special file which the Go compiler will
 	// automatically use when generating Windows binaries to set things like
@@ -463,10 +434,12 @@ func build(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	runPrint("go", args...)
+	args := []string{"build", "-v"}
+	args = appendParameters(args, tags, target.buildPkgs...)
+	runPrint(goCmd, args...)
 }
 
-func appendParameters(args []string, tags []string, target target) []string {
+func appendParameters(args []string, tags []string, pkgs ...string) []string {
 	if pkgdir != "" {
 		args = append(args, "-pkgdir", pkgdir)
 	}
@@ -491,7 +464,7 @@ func appendParameters(args []string, tags []string, target target) []string {
 		args = append(args, "-gcflags", "-N -l")
 	}
 
-	return append(args, target.buildPkg)
+	return append(args, pkgs...)
 }
 
 func buildTar(target target) {
@@ -505,10 +478,7 @@ func buildTar(target target) {
 	}
 
 	build(target, tags)
-
+	codesign(target)
-	if goos == "darwin" {
-		macosCodesign(target.BinaryName())
-	}
 
 	for i := range target.archiveFiles {
 		target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.BinaryName(), 1)
@@ -531,10 +501,7 @@ func buildZip(target target) {
 	}
 
 	build(target, tags)
-
+	codesign(target)
-	if goos == "windows" {
-		windowsCodesign(target.BinaryName())
-	}
 
 	for i := range target.archiveFiles {
 		target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.BinaryName(), 1)
@@ -598,79 +565,55 @@ func buildDeb(target target) {
 	for _, dep := range target.debdeps {
 		args = append(args, "-d", dep)
 	}
+	for _, service := range target.systemdServices {
+		args = append(args, "--deb-systemd", service)
+	}
 	if target.debpost != "" {
 		args = append(args, "--after-upgrade", target.debpost)
 	}
+	if target.debpre != "" {
+		args = append(args, "--before-install", target.debpre)
+	}
 	runPrint("fpm", args...)
 }
 
-func buildSnap(target target) {
+func shouldBuildSyso(dir string) (string, error) {
-	os.RemoveAll("snap")
+	type M map[string]interface{}
-
+	version := getVersion()
-	tmpl, err := template.ParseFiles("snapcraft.yaml.template")
+	version = strings.TrimPrefix(version, "v")
-	if err != nil {
+	major, minor, patch := semanticVersion()
-		log.Fatal(err)
+	bs, err := json.Marshal(M{
-	}
+		"FixedFileInfo": M{
-	f, err := os.Create("snapcraft.yaml")
+			"FileVersion": M{
-	defer f.Close()
+				"Major": major,
-	if err != nil {
+				"Minor": minor,
-		log.Fatal(err)
+				"Patch": patch,
-	}
+			},
-
+			"ProductVersion": M{
-	snaparch := goarch
+				"Major": major,
-	if snaparch == "armhf" {
+				"Minor": minor,
-		goarch = "arm"
+				"Patch": patch,
-	} else if snaparch == "i386" {
+			},
-		goarch = "386"
+		},
-	}
+		"StringFileInfo": M{
-	snapver := version
+			"FileDescription": "Open Source Continuous File Synchronization",
-	if strings.HasPrefix(snapver, "v") {
+			"LegalCopyright":  "The Syncthing Authors",
-		snapver = snapver[1:]
+			"FileVersion":     version,
-	}
+			"ProductVersion":  version,
-	snapgrade := "devel"
+			"ProductName":     "Syncthing",
-	if matched, _ := regexp.MatchString(`^\d+\.\d+\.\d+(-rc.\d+)?$`, snapver); matched {
+		},
-		snapgrade = "stable"
+		"IconPath": "assets/logo.ico",
-	}
-	err = tmpl.Execute(f, map[string]string{
-		"Version":            snapver,
-		"HostArchitecture":   runtime.GOARCH,
-		"TargetArchitecture": snaparch,
-		"Grade":              snapgrade,
 	})
 	if err != nil {
-		log.Fatal(err)
+		return "", err
 	}
-	runPrint("snapcraft", "clean")
-	build(target, []string{"noupgrade"})
-	runPrint("snapcraft")
-}
 
-func shouldBuildSyso(dir string) (string, error) {
 	jsonPath := filepath.Join(dir, "versioninfo.json")
-	file, err := os.Create(filepath.Join(dir, "versioninfo.json"))
+	err = ioutil.WriteFile(jsonPath, bs, 0644)
 	if err != nil {
 		return "", errors.New("failed to create " + jsonPath + ": " + err.Error())
 	}
 
-	major, minor, patch, build := semanticVersion()
-	fmt.Fprintf(file, `{
-    "FixedFileInfo": {
-        "FileVersion": {
-            "Major": %s,
-            "Minor": %s,
-            "Patch": %s,
-            "Build": %s
-        }
-    },
-    "StringFileInfo": {
-        "FileDescription": "Open Source Continuous File Synchronization",
-        "LegalCopyright": "The Syncthing Authors",
-        "ProductVersion": "%s",
-        "ProductName": "Syncthing"
-    },
-    "IconPath": "assets/logo.ico"
-}`, major, minor, patch, build, getVersion())
-	file.Close()
 	defer func() {
 		if err := os.Remove(jsonPath); err != nil {
 			log.Printf("Warning: unable to remove generated %s: %v. Please remove it manually.", jsonPath, err)
@@ -733,6 +676,7 @@ func listFiles(dir string) []string {
 		if err != nil {
 			return err
 		}
+
 		if fi.Mode().IsRegular() {
 			res = append(res, path)
 		}
@@ -743,11 +687,11 @@ func listFiles(dir string) []string {
 
 func rebuildAssets() {
 	os.Setenv("SOURCE_DATE_EPOCH", fmt.Sprint(buildStamp()))
-	runPrint("go", "generate", "github.com/syncthing/syncthing/lib/auto", "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto")
+	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/api/auto", "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto")
 }
 
 func lazyRebuildAssets() {
-	if shouldRebuildAssets("lib/auto/gui.files.go", "gui") || shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.files.go", "cmd/strelaypoolsrv/auto/gui") {
+	if shouldRebuildAssets("lib/api/auto/gui.files.go", "gui") || shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.files.go", "cmd/strelaypoolsrv/auto/gui") {
 		rebuildAssets()
 	}
 }
@@ -779,12 +723,28 @@ func shouldRebuildAssets(target, srcdir string) bool {
 }
 
 func proto() {
-	runPrint("go", "generate", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/stdiscosrv")
+	pv := protobufVersion()
+	dependencyRepos = append(dependencyRepos,
+		dependencyRepo{path: "protobuf", repo: "https://github.com/gogo/protobuf.git", commit: pv},
+	)
+
+	runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
+	os.MkdirAll("repos", 0755)
+	for _, dep := range dependencyRepos {
+		path := filepath.Join("repos", dep.path)
+		if _, err := os.Stat(path); err != nil {
+			runPrintInDir("repos", "git", "clone", dep.repo, dep.path)
+		} else {
+			runPrintInDir(path, "git", "fetch")
+		}
+		runPrintInDir(path, "git", "checkout", dep.commit)
+	}
+	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/stdiscosrv")
 }
 
 func translate() {
 	os.Chdir("gui/default/assets/lang")
-	runPipe("lang-en-new.json", "go", "run", "../../../../script/translate.go", "lang-en.json", "../../../")
+	runPipe("lang-en-new.json", goCmd, "run", "../../../../script/translate.go", "lang-en.json", "../../../")
 	os.Remove("lang-en.json")
 	err := os.Rename("lang-en-new.json", "lang-en.json")
 	if err != nil {
@@ -795,26 +755,19 @@ func translate() {
 
 func transifex() {
 	os.Chdir("gui/default/assets/lang")
-	runPrint("go", "run", "../../../../script/transifexdl.go")
+	runPrint(goCmd, "run", "../../../../script/transifexdl.go")
-}
-
-func clean() {
-	rmr("bin")
-	rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/syncthing", goos, goarch)))
 }
 
 func ldflags() string {
-	sep := '='
+	b := new(strings.Builder)
-	if goVersion > 0 && goVersion < 1.5 {
-		sep = ' '
-	}
-
-	b := new(bytes.Buffer)
 	b.WriteString("-w")
-	fmt.Fprintf(b, " -X main.Version%c%s", sep, version)
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Version=%s", version)
-	fmt.Fprintf(b, " -X main.BuildStamp%c%d", sep, buildStamp())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp=%d", buildStamp())
-	fmt.Fprintf(b, " -X main.BuildUser%c%s", sep, buildUser())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User=%s", buildUser())
-	fmt.Fprintf(b, " -X main.BuildHost%c%s", sep, buildHost())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Host=%s", buildHost())
+	if v := os.Getenv("EXTRA_LDFLAGS"); v != "" {
+		fmt.Fprintf(b, " %s", v)
+	}
 	return b.String()
 }
 
@@ -828,13 +781,7 @@ func rmr(paths ...string) {
 }
 
 func getReleaseVersion() (string, error) {
-	fd, err := os.Open("RELEASE")
+	bs, err := ioutil.ReadFile("RELEASE")
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-
-	bs, err := ioutil.ReadAll(fd)
 	if err != nil {
 		return "", err
 	}
@@ -871,13 +818,18 @@ func getVersion() string {
 	return "unknown-dev"
 }
 
-func semanticVersion() (major, minor, patch, build string) {
+func semanticVersion() (major, minor, patch int) {
-	r := regexp.MustCompile(`v(?P<Major>\d+)\.(?P<Minor>\d+).(?P<Patch>\d+).*\+(?P<CommitsAhead>\d+)`)
+	r := regexp.MustCompile(`v(\d+)\.(\d+).(\d+)`)
 	matches := r.FindStringSubmatch(getVersion())
-	if len(matches) != 5 {
+	if len(matches) != 4 {
-		return "0", "0", "0", "0"
+		return 0, 0, 0
 	}
-	return matches[1], matches[2], matches[3], matches[4]
+
+	var ints [3]int
+	for i, s := range matches[1:] {
+		ints[i], _ = strconv.Atoi(s)
+	}
+	return ints[0], ints[1], ints[2]
 }
 
 func getBranchSuffix() string {
@@ -997,6 +949,10 @@ func runError(cmd string, args ...string) ([]byte, error) {
 }
 
 func runPrint(cmd string, args ...string) {
+	runPrintInDir(".", cmd, args...)
+}
+
+func runPrintInDir(dir string, cmd string, args ...string) {
 	if debug {
 		t0 := time.Now()
 		log.Println("runPrint:", cmd, strings.Join(args, " "))
@@ -1007,6 +963,7 @@ func runPrint(cmd string, args ...string) {
 	ecmd := exec.Command(cmd, args...)
 	ecmd.Stdout = os.Stdout
 	ecmd.Stderr = os.Stderr
+	ecmd.Dir = dir
 	err := ecmd.Run()
 	if err != nil {
 		log.Fatal(err)
@@ -1168,6 +1125,15 @@ func zipFile(out string, files []archiveFile) {
 	}
 }
 
+func codesign(target target) {
+	switch goos {
+	case "windows":
+		windowsCodesign(target.BinaryName())
+	case "darwin":
+		macosCodesign(target.BinaryName())
+	}
+}
+
 func macosCodesign(file string) {
 	if pass := os.Getenv("CODESIGN_KEYCHAIN_PASS"); pass != "" {
 		bs, err := runError("security", "unlock-keychain", "-p", pass)
@@ -1178,7 +1144,7 @@ func macosCodesign(file string) {
 	}
 
 	if id := os.Getenv("CODESIGN_IDENTITY"); id != "" {
-		bs, err := runError("codesign", "-s", id, file)
+		bs, err := runError("codesign", "--options=runtime", "-s", id, file)
 		if err != nil {
 			log.Println("Codesign: signing failed:", string(bs))
 			return
@@ -1226,12 +1192,12 @@ func windowsCodesign(file string) {
 
 func metalint() {
 	lazyRebuildAssets()
-	runPrint("go", "test", "-run", "Metalint", "./meta")
+	runPrint(goCmd, "test", "-run", "Metalint", "./meta")
 }
 
 func metalintShort() {
 	lazyRebuildAssets()
-	runPrint("go", "test", "-short", "-run", "Metalint", "./meta")
+	runPrint(goCmd, "test", "-short", "-run", "Metalint", "./meta")
 }
 
 func temporaryBuildDir() (string, error) {
@@ -1260,93 +1226,17 @@ func temporaryBuildDir() (string, error) {
 	return filepath.Join(tmpDir, base), nil
 }
 
-func buildGOPATH(gopath string) error {
-	pkg := filepath.Join(gopath, "src/github.com/syncthing/syncthing")
-	dirs := []string{"cmd", "gui", "lib", "meta", "script", "test", "vendor"}
-
-	if debug {
-		t0 := time.Now()
-		log.Println("build temporary GOPATH in", gopath)
-		defer func() {
-			log.Println("... in", time.Since(t0))
-		}()
-	}
-
-	// Walk the sources and copy the files into the temporary GOPATH.
-	// Remember which files are supposed to be present so we can clean
-	// out everything else in the next step. The copyFile() step will
-	// only actually copy the file if it doesn't exist or the contents
-	// differ.
-
-	exists := map[string]struct{}{}
-	for _, dir := range dirs {
-		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			if info.IsDir() {
-				return nil
-			}
-
-			dst := filepath.Join(pkg, path)
-			exists[dst] = struct{}{}
-
-			if err := copyFile(path, dst, info.Mode()); err != nil {
-				return err
-			}
-
-			return nil
-		})
-		if err != nil {
-			return err
-		}
-	}
-
-	// Walk the temporary GOPATH and remove any files that we wouldn't
-	// have copied there in the previous step.
-
-	filepath.Walk(pkg, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if info.IsDir() {
-			return nil
-		}
-		if _, ok := exists[path]; !ok {
-			os.Remove(path)
-		}
-		return nil
-	})
-
-	return nil
-}
-
-func gopath() string {
-	if gopath := os.Getenv("GOPATH"); gopath != "" {
-		// The env var is set, use that.
-		return gopath
-	}
-
-	// Ask Go what it thinks.
-	bs, err := runError("go", "env", "GOPATH")
-	if err != nil {
-		return ""
-	}
-
-	// We got something. Check if we are in fact available in that location.
-	gopath := string(bs)
-	if _, err := os.Stat(filepath.Join(gopath, "src/github.com/syncthing/syncthing/build.go")); err == nil {
-		// That seems to be the gopath.
-		return gopath
-	}
-
-	// The gopath is not valid.
-	return ""
-}
-
 func (t target) BinaryName() string {
 	if goos == "windows" {
 		return t.binaryName + ".exe"
 	}
 	return t.binaryName
 }
+
+func protobufVersion() string {
+	bs, err := runError(goCmd, "list", "-f", "{{.Version}}", "-m", "github.com/gogo/protobuf")
+	if err != nil {
+		log.Fatal("Getting protobuf version:", err)
+	}
+	return string(bs)
+}

cmd/stbench/main.go

@@ -1,143 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// This doesn't build on Windows due to the Rusage stuff.
-
-// +build !windows
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"runtime"
-	"syscall"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/rc"
-)
-
-var homeDir = "h1"
-var syncthingBin = "./bin/syncthing"
-var test = "scan"
-
-func main() {
-	flag.StringVar(&homeDir, "home", homeDir, "Home directory location")
-	flag.StringVar(&syncthingBin, "bin", syncthingBin, "Binary location")
-	flag.StringVar(&test, "test", test, "Test to run")
-	flag.Parse()
-
-	switch test {
-	case "scan":
-		// scan measures the resource usage required to perform the initial
-		// scan, without cleaning away the database first.
-		testScan()
-	}
-}
-
-// testScan starts a process and reports on the resource usage required to
-// perform the initial scan.
-func testScan() {
-	log.Println("Starting...")
-	p := rc.NewProcess("127.0.0.1:8081")
-	if err := p.Start(syncthingBin, "-home", homeDir, "-no-browser"); err != nil {
-		log.Println(err)
-		return
-	}
-	defer p.Stop()
-
-	wallTime := awaitScanComplete(p)
-
-	report(p, wallTime)
-}
-
-// awaitScanComplete waits for a folder to transition idle->scanning and
-// then scanning->idle and returns the time taken for the scan.
-func awaitScanComplete(p *rc.Process) time.Duration {
-	log.Println("Awaiting scan completion...")
-	var t0, t1 time.Time
-	lastEvent := 0
-loop:
-	for {
-		evs, err := p.Events(lastEvent)
-		if err != nil {
-			continue
-		}
-
-		for _, ev := range evs {
-			if ev.Type == "StateChanged" {
-				data := ev.Data.(map[string]interface{})
-				log.Println(ev)
-
-				if data["to"].(string) == "scanning" {
-					t0 = ev.Time
-					continue
-				}
-
-				if !t0.IsZero() && data["to"].(string) == "idle" {
-					t1 = ev.Time
-					break loop
-				}
-			}
-			lastEvent = ev.ID
-		}
-
-		time.Sleep(250 * time.Millisecond)
-	}
-
-	return t1.Sub(t0)
-}
-
-// report stops the given process and reports on its resource usage in two
-// ways: human readable to stderr, and CSV to stdout.
-func report(p *rc.Process, wallTime time.Duration) {
-	sv, err := p.SystemVersion()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	ss, err := p.SystemStatus()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	proc, err := p.Stop()
-	if err != nil {
-		return
-	}
-
-	rusage, ok := proc.SysUsage().(*syscall.Rusage)
-	if !ok {
-		return
-	}
-
-	log.Println("Version:", sv.Version)
-	log.Println("Alloc:", ss.Alloc/1024, "KiB")
-	log.Println("Sys:", ss.Sys/1024, "KiB")
-	log.Println("Goroutines:", ss.Goroutines)
-	log.Println("Wall time:", wallTime)
-	log.Println("Utime:", time.Duration(rusage.Utime.Nano()))
-	log.Println("Stime:", time.Duration(rusage.Stime.Nano()))
-	if runtime.GOOS == "darwin" {
-		// Darwin reports in bytes, Linux seems to report in KiB even
-		// though the manpage says otherwise.
-		rusage.Maxrss /= 1024
-	}
-	log.Println("MaxRSS:", rusage.Maxrss, "KiB")
-
-	fmt.Printf("%s,%d,%d,%d,%.02f,%.02f,%.02f,%d\n",
-		sv.Version,
-		ss.Alloc/1024,
-		ss.Sys/1024,
-		ss.Goroutines,
-		wallTime.Seconds(),
-		time.Duration(rusage.Utime.Nano()).Seconds(),
-		time.Duration(rusage.Stime.Nano()).Seconds(),
-		rusage.Maxrss)
-}

cmd/stcli/LICENSE

@@ -1,19 +0,0 @@
-Copyright (C) 2014 Audrius Butkevičius
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-- The above copyright notice and this permission notice shall be included in
-  all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

cmd/stcli/client.go

@@ -1,115 +1,96 @@
-// Copyright (C) 2014 Audrius Butkevičius
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
 	"net/http"
 	"strings"
 
-	"github.com/AudriusButkevicius/cli"
+	"github.com/syncthing/syncthing/lib/config"
 )
 
 type APIClient struct {
-	httpClient http.Client
+	http.Client
-	endpoint   string
+	cfg    config.GUIConfiguration
-	apikey     string
+	apikey string
-	username   string
-	password   string
-	id         string
-	csrf       string
 }
 
-var instance *APIClient
+func getClient(cfg config.GUIConfiguration) *APIClient {
-
-func getClient(c *cli.Context) *APIClient {
-	if instance != nil {
-		return instance
-	}
-	endpoint := c.GlobalString("endpoint")
-	if !strings.HasPrefix(endpoint, "http") {
-		endpoint = "http://" + endpoint
-	}
 	httpClient := http.Client{
 		Transport: &http.Transport{
 			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: c.GlobalBool("insecure"),
+				InsecureSkipVerify: true,
+			},
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial(cfg.Network(), cfg.Address())
 			},
 		},
 	}
-	client := APIClient{
+	return &APIClient{
-		httpClient: httpClient,
+		Client: httpClient,
-		endpoint:   endpoint,
+		cfg:    cfg,
-		apikey:     c.GlobalString("apikey"),
+		apikey: cfg.APIKey,
-		username:   c.GlobalString("username"),
-		password:   c.GlobalString("password"),
 	}
-
-	if client.apikey == "" {
-		request, err := http.NewRequest("GET", client.endpoint, nil)
-		die(err)
-		response := client.handleRequest(request)
-		client.id = response.Header.Get("X-Syncthing-ID")
-		if client.id == "" {
-			die("Failed to get device ID")
-		}
-		for _, item := range response.Cookies() {
-			if item.Name == "CSRF-Token-"+client.id[:5] {
-				client.csrf = item.Value
-				goto csrffound
-			}
-		}
-		die("Failed to get CSRF token")
-	csrffound:
-	}
-	instance = &client
-	return &client
 }
 
-func (client *APIClient) handleRequest(request *http.Request) *http.Response {
+func (c *APIClient) Endpoint() string {
-	if client.apikey != "" {
+	if c.cfg.Network() == "unix" {
-		request.Header.Set("X-API-Key", client.apikey)
+		return "http://unix/"
 	}
-	if client.username != "" || client.password != "" {
+	url := c.cfg.URL()
-		request.SetBasicAuth(client.username, client.password)
+	if !strings.HasSuffix(url, "/") {
-	}
+		url += "/"
-	if client.csrf != "" {
-		request.Header.Set("X-CSRF-Token-"+client.id[:5], client.csrf)
 	}
+	return url
+}
 
-	response, err := client.httpClient.Do(request)
+func (c *APIClient) Do(req *http.Request) (*http.Response, error) {
-	die(err)
+	req.Header.Set("X-API-Key", c.apikey)
+	resp, err := c.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	return resp, checkResponse(resp)
+}
 
+func (c *APIClient) Get(url string) (*http.Response, error) {
+	request, err := http.NewRequest("GET", c.Endpoint()+"rest/"+url, nil)
+	if err != nil {
+		return nil, err
+	}
+	return c.Do(request)
+}
+
+func (c *APIClient) Post(url, body string) (*http.Response, error) {
+	request, err := http.NewRequest("POST", c.Endpoint()+"rest/"+url, bytes.NewBufferString(body))
+	if err != nil {
+		return nil, err
+	}
+	return c.Do(request)
+}
+
+func checkResponse(response *http.Response) error {
 	if response.StatusCode == 404 {
-		die("Invalid endpoint or API call")
+		return errors.New("invalid endpoint or API call")
-	} else if response.StatusCode == 401 {
-		die("Invalid username or password")
 	} else if response.StatusCode == 403 {
-		if client.apikey == "" {
+		return errors.New("invalid API key")
-			die("Invalid CSRF token")
-		}
-		die("Invalid API key")
 	} else if response.StatusCode != 200 {
-		body := strings.TrimSpace(string(responseToBArray(response)))
+		data, err := responseToBArray(response)
-		if body != "" {
+		if err != nil {
-			die(body)
+			return err
 		}
-		die("Unknown HTTP status returned: " + response.Status)
+		body := strings.TrimSpace(string(data))
+		return fmt.Errorf("unexpected HTTP status returned: %s\n%s", response.Status, body)
 	}
-	return response
+	return nil
-}
-
-func httpGet(c *cli.Context, url string) *http.Response {
-	client := getClient(c)
-	request, err := http.NewRequest("GET", client.endpoint+"/rest/"+url, nil)
-	die(err)
-	return client.handleRequest(request)
-}
-
-func httpPost(c *cli.Context, url string, body string) *http.Response {
-	client := getClient(c)
-	request, err := http.NewRequest("POST", client.endpoint+"/rest/"+url, bytes.NewBufferString(body))
-	die(err)
-	return client.handleRequest(request)
 }

cmd/stcli/cmd_devices.go

@@ -1,188 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-	"github.com/syncthing/syncthing/lib/config"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "devices",
-		HideHelp: true,
-		Usage:    "Device command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "list",
-				Usage:    "List registered devices",
-				Requires: &cli.Requires{},
-				Action:   devicesList,
-			},
-			{
-				Name:     "add",
-				Usage:    "Add a new device",
-				Requires: &cli.Requires{"device id", "device name?"},
-				Action:   devicesAdd,
-			},
-			{
-				Name:     "remove",
-				Usage:    "Remove an existing device",
-				Requires: &cli.Requires{"device id"},
-				Action:   devicesRemove,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a property of a device",
-				Requires: &cli.Requires{"device id", "property"},
-				Action:   devicesGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a property of a device",
-				Requires: &cli.Requires{"device id", "property", "value..."},
-				Action:   devicesSet,
-			},
-		},
-	})
-}
-
-func devicesList(c *cli.Context) {
-	cfg := getConfig(c)
-	first := true
-	writer := newTableWriter()
-	for _, device := range cfg.Devices {
-		if !first {
-			fmt.Fprintln(writer)
-		}
-		fmt.Fprintln(writer, "ID:\t", device.DeviceID, "\t")
-		fmt.Fprintln(writer, "Name:\t", device.Name, "\t(name)")
-		fmt.Fprintln(writer, "Address:\t", strings.Join(device.Addresses, " "), "\t(address)")
-		fmt.Fprintln(writer, "Compression:\t", device.Compression, "\t(compression)")
-		fmt.Fprintln(writer, "Certificate name:\t", device.CertName, "\t(certname)")
-		fmt.Fprintln(writer, "Introducer:\t", device.Introducer, "\t(introducer)")
-		first = false
-	}
-	writer.Flush()
-}
-
-func devicesAdd(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-
-	newDevice := config.DeviceConfiguration{
-		DeviceID:  id,
-		Name:      nid,
-		Addresses: []string{"dynamic"},
-	}
-
-	if len(c.Args()) > 1 {
-		newDevice.Name = c.Args()[1]
-	}
-
-	if len(c.Args()) > 2 {
-		addresses := c.Args()[2:]
-		for _, item := range addresses {
-			if item == "dynamic" {
-				continue
-			}
-			validAddress(item)
-		}
-		newDevice.Addresses = addresses
-	}
-
-	cfg := getConfig(c)
-	for _, device := range cfg.Devices {
-		if device.DeviceID == id {
-			die("Device " + nid + " already exists")
-		}
-	}
-	cfg.Devices = append(cfg.Devices, newDevice)
-	setConfig(c, cfg)
-}
-
-func devicesRemove(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-	if nid == getMyID(c) {
-		die("Cannot remove yourself")
-	}
-	cfg := getConfig(c)
-	for i, device := range cfg.Devices {
-		if device.DeviceID == id {
-			last := len(cfg.Devices) - 1
-			cfg.Devices[i] = cfg.Devices[last]
-			cfg.Devices = cfg.Devices[:last]
-			setConfig(c, cfg)
-			return
-		}
-	}
-	die("Device " + nid + " not found")
-}
-
-func devicesGet(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-	arg := c.Args()[1]
-	cfg := getConfig(c)
-	for _, device := range cfg.Devices {
-		if device.DeviceID != id {
-			continue
-		}
-		switch strings.ToLower(arg) {
-		case "name":
-			fmt.Println(device.Name)
-		case "address":
-			fmt.Println(strings.Join(device.Addresses, "\n"))
-		case "compression":
-			fmt.Println(device.Compression.String())
-		case "certname":
-			fmt.Println(device.CertName)
-		case "introducer":
-			fmt.Println(device.Introducer)
-		default:
-			die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
-		}
-		return
-	}
-	die("Device " + nid + " not found")
-}
-
-func devicesSet(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-	arg := c.Args()[1]
-	config := getConfig(c)
-	for i, device := range config.Devices {
-		if device.DeviceID != id {
-			continue
-		}
-		switch strings.ToLower(arg) {
-		case "name":
-			config.Devices[i].Name = strings.Join(c.Args()[2:], " ")
-		case "address":
-			for _, item := range c.Args()[2:] {
-				if item == "dynamic" {
-					continue
-				}
-				validAddress(item)
-			}
-			config.Devices[i].Addresses = c.Args()[2:]
-		case "compression":
-			err := config.Devices[i].Compression.UnmarshalText([]byte(c.Args()[2]))
-			die(err)
-		case "certname":
-			config.Devices[i].CertName = strings.Join(c.Args()[2:], " ")
-		case "introducer":
-			config.Devices[i].Introducer = parseBool(c.Args()[2])
-		default:
-			die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
-		}
-		setConfig(c, config)
-		return
-	}
-	die("Device " + nid + " not found")
-}

cmd/stcli/cmd_errors.go

@@ -1,67 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "errors",
-		HideHelp: true,
-		Usage:    "Error command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "show",
-				Usage:    "Show pending errors",
-				Requires: &cli.Requires{},
-				Action:   errorsShow,
-			},
-			{
-				Name:     "push",
-				Usage:    "Push an error to active clients",
-				Requires: &cli.Requires{"error message..."},
-				Action:   errorsPush,
-			},
-			{
-				Name:     "clear",
-				Usage:    "Clear pending errors",
-				Requires: &cli.Requires{},
-				Action:   wrappedHTTPPost("system/error/clear"),
-			},
-		},
-	})
-}
-
-func errorsShow(c *cli.Context) {
-	response := httpGet(c, "system/error")
-	var data map[string][]map[string]interface{}
-	json.Unmarshal(responseToBArray(response), &data)
-	writer := newTableWriter()
-	for _, item := range data["errors"] {
-		time := item["when"].(string)[:19]
-		time = strings.Replace(time, "T", " ", 1)
-		err := item["message"].(string)
-		err = strings.TrimSpace(err)
-		fmt.Fprintln(writer, time+":\t"+err)
-	}
-	writer.Flush()
-}
-
-func errorsPush(c *cli.Context) {
-	err := strings.Join(c.Args(), " ")
-	response := httpPost(c, "system/error", strings.TrimSpace(err))
-	if response.StatusCode != 200 {
-		err = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
-		body := string(responseToBArray(response))
-		if body != "" {
-			err += "\nBody: " + body
-		}
-		die(err)
-	}
-}

cmd/stcli/cmd_folders.go

@@ -1,361 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"path/filepath"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/fs"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "folders",
-		HideHelp: true,
-		Usage:    "Folder command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "list",
-				Usage:    "List available folders",
-				Requires: &cli.Requires{},
-				Action:   foldersList,
-			},
-			{
-				Name:     "add",
-				Usage:    "Add a new folder",
-				Requires: &cli.Requires{"folder id", "directory"},
-				Action:   foldersAdd,
-			},
-			{
-				Name:     "remove",
-				Usage:    "Remove an existing folder",
-				Requires: &cli.Requires{"folder id"},
-				Action:   foldersRemove,
-			},
-			{
-				Name:     "override",
-				Usage:    "Override changes from other nodes for a master folder",
-				Requires: &cli.Requires{"folder id"},
-				Action:   foldersOverride,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a property of a folder",
-				Requires: &cli.Requires{"folder id", "property"},
-				Action:   foldersGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a property of a folder",
-				Requires: &cli.Requires{"folder id", "property", "value..."},
-				Action:   foldersSet,
-			},
-			{
-				Name:     "unset",
-				Usage:    "Unset a property of a folder",
-				Requires: &cli.Requires{"folder id", "property"},
-				Action:   foldersUnset,
-			},
-			{
-				Name:     "devices",
-				Usage:    "Folder devices command group",
-				HideHelp: true,
-				Subcommands: []cli.Command{
-					{
-						Name:     "list",
-						Usage:    "List of devices which the folder is shared with",
-						Requires: &cli.Requires{"folder id"},
-						Action:   foldersDevicesList,
-					},
-					{
-						Name:     "add",
-						Usage:    "Share a folder with a device",
-						Requires: &cli.Requires{"folder id", "device id"},
-						Action:   foldersDevicesAdd,
-					},
-					{
-						Name:     "remove",
-						Usage:    "Unshare a folder with a device",
-						Requires: &cli.Requires{"folder id", "device id"},
-						Action:   foldersDevicesRemove,
-					},
-					{
-						Name:     "clear",
-						Usage:    "Unshare a folder with all devices",
-						Requires: &cli.Requires{"folder id"},
-						Action:   foldersDevicesClear,
-					},
-				},
-			},
-		},
-	})
-}
-
-func foldersList(c *cli.Context) {
-	cfg := getConfig(c)
-	first := true
-	writer := newTableWriter()
-	for _, folder := range cfg.Folders {
-		if !first {
-			fmt.Fprintln(writer)
-		}
-		fs := folder.Filesystem()
-		fmt.Fprintln(writer, "ID:\t", folder.ID, "\t")
-		fmt.Fprintln(writer, "Path:\t", fs.URI(), "\t(directory)")
-		fmt.Fprintln(writer, "Path type:\t", fs.Type(), "\t(directory-type)")
-		fmt.Fprintln(writer, "Folder type:\t", folder.Type, "\t(type)")
-		fmt.Fprintln(writer, "Ignore permissions:\t", folder.IgnorePerms, "\t(permissions)")
-		fmt.Fprintln(writer, "Rescan interval in seconds:\t", folder.RescanIntervalS, "\t(rescan)")
-
-		if folder.Versioning.Type != "" {
-			fmt.Fprintln(writer, "Versioning:\t", folder.Versioning.Type, "\t(versioning)")
-			for key, value := range folder.Versioning.Params {
-				fmt.Fprintf(writer, "Versioning %s:\t %s \t(versioning-%s)\n", key, value, key)
-			}
-		}
-		first = false
-	}
-	writer.Flush()
-}
-
-func foldersAdd(c *cli.Context) {
-	cfg := getConfig(c)
-	abs, err := filepath.Abs(c.Args()[1])
-	die(err)
-	folder := config.FolderConfiguration{
-		ID:             c.Args()[0],
-		Path:           filepath.Clean(abs),
-		FilesystemType: fs.FilesystemTypeBasic,
-	}
-	cfg.Folders = append(cfg.Folders, folder)
-	setConfig(c, cfg)
-}
-
-func foldersRemove(c *cli.Context) {
-	cfg := getConfig(c)
-	rid := c.Args()[0]
-	for i, folder := range cfg.Folders {
-		if folder.ID == rid {
-			last := len(cfg.Folders) - 1
-			cfg.Folders[i] = cfg.Folders[last]
-			cfg.Folders = cfg.Folders[:last]
-			setConfig(c, cfg)
-			return
-		}
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersOverride(c *cli.Context) {
-	cfg := getConfig(c)
-	rid := c.Args()[0]
-	for _, folder := range cfg.Folders {
-		if folder.ID == rid && folder.Type == config.FolderTypeSendOnly {
-			response := httpPost(c, "db/override", "")
-			if response.StatusCode != 200 {
-				err := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
-				body := string(responseToBArray(response))
-				if body != "" {
-					err += "\nBody: " + body
-				}
-				die(err)
-			}
-			return
-		}
-	}
-	die("Folder " + rid + " not found or folder not master")
-}
-
-func foldersGet(c *cli.Context) {
-	cfg := getConfig(c)
-	rid := c.Args()[0]
-	arg := strings.ToLower(c.Args()[1])
-	for _, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		if strings.HasPrefix(arg, "versioning-") {
-			arg = arg[11:]
-			value, ok := folder.Versioning.Params[arg]
-			if ok {
-				fmt.Println(value)
-				return
-			}
-			die("Versioning property " + c.Args()[1][11:] + " not found")
-		}
-		switch arg {
-		case "directory":
-			fmt.Println(folder.Filesystem().URI())
-		case "directory-type":
-			fmt.Println(folder.Filesystem().Type())
-		case "type":
-			fmt.Println(folder.Type)
-		case "permissions":
-			fmt.Println(folder.IgnorePerms)
-		case "rescan":
-			fmt.Println(folder.RescanIntervalS)
-		case "versioning":
-			if folder.Versioning.Type != "" {
-				fmt.Println(folder.Versioning.Type)
-			}
-		default:
-			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, directory-type, type, permissions, versioning, versioning-<key>")
-		}
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersSet(c *cli.Context) {
-	rid := c.Args()[0]
-	arg := strings.ToLower(c.Args()[1])
-	val := strings.Join(c.Args()[2:], " ")
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		if strings.HasPrefix(arg, "versioning-") {
-			cfg.Folders[i].Versioning.Params[arg[11:]] = val
-			setConfig(c, cfg)
-			return
-		}
-		switch arg {
-		case "directory":
-			cfg.Folders[i].Path = val
-		case "directory-type":
-			var fsType fs.FilesystemType
-			fsType.UnmarshalText([]byte(val))
-			cfg.Folders[i].FilesystemType = fsType
-		case "type":
-			var t config.FolderType
-			if err := t.UnmarshalText([]byte(val)); err != nil {
-				die("Invalid folder type: " + err.Error())
-			}
-			cfg.Folders[i].Type = t
-		case "permissions":
-			cfg.Folders[i].IgnorePerms = parseBool(val)
-		case "rescan":
-			cfg.Folders[i].RescanIntervalS = parseInt(val)
-		case "versioning":
-			cfg.Folders[i].Versioning.Type = val
-		default:
-			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, master, permissions, versioning, versioning-<key>")
-		}
-		setConfig(c, cfg)
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersUnset(c *cli.Context) {
-	rid := c.Args()[0]
-	arg := strings.ToLower(c.Args()[1])
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		if strings.HasPrefix(arg, "versioning-") {
-			arg = arg[11:]
-			if _, ok := folder.Versioning.Params[arg]; ok {
-				delete(cfg.Folders[i].Versioning.Params, arg)
-				setConfig(c, cfg)
-				return
-			}
-			die("Versioning property " + c.Args()[1][11:] + " not found")
-		}
-		switch arg {
-		case "versioning":
-			cfg.Folders[i].Versioning.Type = ""
-			cfg.Folders[i].Versioning.Params = make(map[string]string)
-		default:
-			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: versioning, versioning-<key>")
-		}
-		setConfig(c, cfg)
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesList(c *cli.Context) {
-	rid := c.Args()[0]
-	cfg := getConfig(c)
-	for _, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		for _, device := range folder.Devices {
-			fmt.Println(device.DeviceID)
-		}
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesAdd(c *cli.Context) {
-	rid := c.Args()[0]
-	nid := parseDeviceID(c.Args()[1])
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		for _, device := range folder.Devices {
-			if device.DeviceID == nid {
-				die("Device " + c.Args()[1] + " is already part of this folder")
-			}
-		}
-		for _, device := range cfg.Devices {
-			if device.DeviceID == nid {
-				cfg.Folders[i].Devices = append(folder.Devices, config.FolderDeviceConfiguration{
-					DeviceID: device.DeviceID,
-				})
-				setConfig(c, cfg)
-				return
-			}
-		}
-		die("Device " + c.Args()[1] + " not found in device list")
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesRemove(c *cli.Context) {
-	rid := c.Args()[0]
-	nid := parseDeviceID(c.Args()[1])
-	cfg := getConfig(c)
-	for ri, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		for ni, device := range folder.Devices {
-			if device.DeviceID == nid {
-				last := len(folder.Devices) - 1
-				cfg.Folders[ri].Devices[ni] = folder.Devices[last]
-				cfg.Folders[ri].Devices = cfg.Folders[ri].Devices[:last]
-				setConfig(c, cfg)
-				return
-			}
-		}
-		die("Device " + c.Args()[1] + " not found")
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesClear(c *cli.Context) {
-	rid := c.Args()[0]
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		cfg.Folders[i].Devices = []config.FolderDeviceConfiguration{}
-		setConfig(c, cfg)
-		return
-	}
-	die("Folder " + rid + " not found")
-}

cmd/stcli/cmd_general.go

@@ -1,94 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, []cli.Command{
-		{
-			Name:     "id",
-			Usage:    "Get ID of the Syncthing client",
-			Requires: &cli.Requires{},
-			Action:   generalID,
-		},
-		{
-			Name:     "status",
-			Usage:    "Configuration status, whether or not a restart is required for changes to take effect",
-			Requires: &cli.Requires{},
-			Action:   generalStatus,
-		},
-		{
-			Name:     "config",
-			Usage:    "Configuration",
-			Requires: &cli.Requires{},
-			Action:   generalConfiguration,
-		},
-		{
-			Name:     "restart",
-			Usage:    "Restart syncthing",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/restart"),
-		},
-		{
-			Name:     "shutdown",
-			Usage:    "Shutdown syncthing",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/shutdown"),
-		},
-		{
-			Name:     "reset",
-			Usage:    "Reset syncthing deleting all folders and devices",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/reset"),
-		},
-		{
-			Name:     "upgrade",
-			Usage:    "Upgrade syncthing (if a newer version is available)",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/upgrade"),
-		},
-		{
-			Name:     "version",
-			Usage:    "Syncthing client version",
-			Requires: &cli.Requires{},
-			Action:   generalVersion,
-		},
-	}...)
-}
-
-func generalID(c *cli.Context) {
-	fmt.Println(getMyID(c))
-}
-
-func generalStatus(c *cli.Context) {
-	response := httpGet(c, "system/config/insync")
-	var status struct{ ConfigInSync bool }
-	json.Unmarshal(responseToBArray(response), &status)
-	if !status.ConfigInSync {
-		die("Config out of sync")
-	}
-	fmt.Println("Config in sync")
-}
-
-func generalConfiguration(c *cli.Context) {
-	response := httpGet(c, "system/config")
-	var jsResponse interface{}
-	json.Unmarshal(responseToBArray(response), &jsResponse)
-	enc := json.NewEncoder(os.Stdout)
-	enc.SetIndent("", "  ")
-	enc.Encode(jsResponse)
-}
-
-func generalVersion(c *cli.Context) {
-	response := httpGet(c, "system/version")
-	version := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &version)
-	prettyPrintJSON(version)
-}

cmd/stcli/cmd_gui.go

@@ -1,127 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "gui",
-		HideHelp: true,
-		Usage:    "GUI command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "dump",
-				Usage:    "Show all GUI configuration settings",
-				Requires: &cli.Requires{},
-				Action:   guiDump,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a GUI configuration setting",
-				Requires: &cli.Requires{"setting"},
-				Action:   guiGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a GUI configuration setting",
-				Requires: &cli.Requires{"setting", "value"},
-				Action:   guiSet,
-			},
-			{
-				Name:     "unset",
-				Usage:    "Unset a GUI configuration setting",
-				Requires: &cli.Requires{"setting"},
-				Action:   guiUnset,
-			},
-		},
-	})
-}
-
-func guiDump(c *cli.Context) {
-	cfg := getConfig(c).GUI
-	writer := newTableWriter()
-	fmt.Fprintln(writer, "Enabled:\t", cfg.Enabled, "\t(enabled)")
-	fmt.Fprintln(writer, "Use HTTPS:\t", cfg.UseTLS(), "\t(tls)")
-	fmt.Fprintln(writer, "Listen Addresses:\t", cfg.Address(), "\t(address)")
-	if cfg.User != "" {
-		fmt.Fprintln(writer, "Authentication User:\t", cfg.User, "\t(username)")
-		fmt.Fprintln(writer, "Authentication Password:\t", cfg.Password, "\t(password)")
-	}
-	if cfg.APIKey != "" {
-		fmt.Fprintln(writer, "API Key:\t", cfg.APIKey, "\t(apikey)")
-	}
-	writer.Flush()
-}
-
-func guiGet(c *cli.Context) {
-	cfg := getConfig(c).GUI
-	arg := c.Args()[0]
-	switch strings.ToLower(arg) {
-	case "enabled":
-		fmt.Println(cfg.Enabled)
-	case "tls":
-		fmt.Println(cfg.UseTLS())
-	case "address":
-		fmt.Println(cfg.Address())
-	case "user":
-		if cfg.User != "" {
-			fmt.Println(cfg.User)
-		}
-	case "password":
-		if cfg.User != "" {
-			fmt.Println(cfg.Password)
-		}
-	case "apikey":
-		if cfg.APIKey != "" {
-			fmt.Println(cfg.APIKey)
-		}
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
-	}
-}
-
-func guiSet(c *cli.Context) {
-	cfg := getConfig(c)
-	arg := c.Args()[0]
-	val := c.Args()[1]
-	switch strings.ToLower(arg) {
-	case "enabled":
-		cfg.GUI.Enabled = parseBool(val)
-	case "tls":
-		cfg.GUI.RawUseTLS = parseBool(val)
-	case "address":
-		validAddress(val)
-		cfg.GUI.RawAddress = val
-	case "user":
-		cfg.GUI.User = val
-	case "password":
-		cfg.GUI.Password = val
-	case "apikey":
-		cfg.GUI.APIKey = val
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
-	}
-	setConfig(c, cfg)
-}
-
-func guiUnset(c *cli.Context) {
-	cfg := getConfig(c)
-	arg := c.Args()[0]
-	switch strings.ToLower(arg) {
-	case "user":
-		cfg.GUI.User = ""
-	case "password":
-		cfg.GUI.Password = ""
-	case "apikey":
-		cfg.GUI.APIKey = ""
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: user, password, apikey")
-	}
-	setConfig(c, cfg)
-}

cmd/stcli/cmd_options.go

@@ -1,173 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "options",
-		HideHelp: true,
-		Usage:    "Options command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "dump",
-				Usage:    "Show all Syncthing option settings",
-				Requires: &cli.Requires{},
-				Action:   optionsDump,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a Syncthing option setting",
-				Requires: &cli.Requires{"setting"},
-				Action:   optionsGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a Syncthing option setting",
-				Requires: &cli.Requires{"setting", "value..."},
-				Action:   optionsSet,
-			},
-		},
-	})
-}
-
-func optionsDump(c *cli.Context) {
-	cfg := getConfig(c).Options
-	writer := newTableWriter()
-
-	fmt.Fprintln(writer, "Sync protocol listen addresses:\t", strings.Join(cfg.ListenAddresses, " "), "\t(addresses)")
-	fmt.Fprintln(writer, "Global discovery enabled:\t", cfg.GlobalAnnEnabled, "\t(globalannenabled)")
-	fmt.Fprintln(writer, "Global discovery servers:\t", strings.Join(cfg.GlobalAnnServers, " "), "\t(globalannserver)")
-
-	fmt.Fprintln(writer, "Local discovery enabled:\t", cfg.LocalAnnEnabled, "\t(localannenabled)")
-	fmt.Fprintln(writer, "Local discovery port:\t", cfg.LocalAnnPort, "\t(localannport)")
-
-	fmt.Fprintln(writer, "Outgoing rate limit in KiB/s:\t", cfg.MaxSendKbps, "\t(maxsend)")
-	fmt.Fprintln(writer, "Incoming rate limit in KiB/s:\t", cfg.MaxRecvKbps, "\t(maxrecv)")
-	fmt.Fprintln(writer, "Reconnect interval in seconds:\t", cfg.ReconnectIntervalS, "\t(reconnect)")
-	fmt.Fprintln(writer, "Start browser:\t", cfg.StartBrowser, "\t(browser)")
-	fmt.Fprintln(writer, "Enable UPnP:\t", cfg.NATEnabled, "\t(nat)")
-	fmt.Fprintln(writer, "UPnP Lease in minutes:\t", cfg.NATLeaseM, "\t(natlease)")
-	fmt.Fprintln(writer, "UPnP Renewal period in minutes:\t", cfg.NATRenewalM, "\t(natrenew)")
-	fmt.Fprintln(writer, "Restart on Wake Up:\t", cfg.RestartOnWakeup, "\t(wake)")
-
-	reporting := "unrecognized value"
-	switch cfg.URAccepted {
-	case -1:
-		reporting = "false"
-	case 0:
-		reporting = "undecided/false"
-	case 1:
-		reporting = "true"
-	}
-	fmt.Fprintln(writer, "Anonymous usage reporting:\t", reporting, "\t(reporting)")
-
-	writer.Flush()
-}
-
-func optionsGet(c *cli.Context) {
-	cfg := getConfig(c).Options
-	arg := c.Args()[0]
-	switch strings.ToLower(arg) {
-	case "address":
-		fmt.Println(strings.Join(cfg.ListenAddresses, "\n"))
-	case "globalannenabled":
-		fmt.Println(cfg.GlobalAnnEnabled)
-	case "globalannservers":
-		fmt.Println(strings.Join(cfg.GlobalAnnServers, "\n"))
-	case "localannenabled":
-		fmt.Println(cfg.LocalAnnEnabled)
-	case "localannport":
-		fmt.Println(cfg.LocalAnnPort)
-	case "maxsend":
-		fmt.Println(cfg.MaxSendKbps)
-	case "maxrecv":
-		fmt.Println(cfg.MaxRecvKbps)
-	case "reconnect":
-		fmt.Println(cfg.ReconnectIntervalS)
-	case "browser":
-		fmt.Println(cfg.StartBrowser)
-	case "nat":
-		fmt.Println(cfg.NATEnabled)
-	case "natlease":
-		fmt.Println(cfg.NATLeaseM)
-	case "natrenew":
-		fmt.Println(cfg.NATRenewalM)
-	case "reporting":
-		switch cfg.URAccepted {
-		case -1:
-			fmt.Println("false")
-		case 0:
-			fmt.Println("undecided/false")
-		case 1:
-			fmt.Println("true")
-		default:
-			fmt.Println("unknown")
-		}
-	case "wake":
-		fmt.Println(cfg.RestartOnWakeup)
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
-	}
-}
-
-func optionsSet(c *cli.Context) {
-	config := getConfig(c)
-	arg := c.Args()[0]
-	val := c.Args()[1]
-	switch strings.ToLower(arg) {
-	case "address":
-		for _, item := range c.Args().Tail() {
-			validAddress(item)
-		}
-		config.Options.ListenAddresses = c.Args().Tail()
-	case "globalannenabled":
-		config.Options.GlobalAnnEnabled = parseBool(val)
-	case "globalannserver":
-		for _, item := range c.Args().Tail() {
-			validAddress(item)
-		}
-		config.Options.GlobalAnnServers = c.Args().Tail()
-	case "localannenabled":
-		config.Options.LocalAnnEnabled = parseBool(val)
-	case "localannport":
-		config.Options.LocalAnnPort = parsePort(val)
-	case "maxsend":
-		config.Options.MaxSendKbps = parseUint(val)
-	case "maxrecv":
-		config.Options.MaxRecvKbps = parseUint(val)
-	case "reconnect":
-		config.Options.ReconnectIntervalS = parseUint(val)
-	case "browser":
-		config.Options.StartBrowser = parseBool(val)
-	case "nat":
-		config.Options.NATEnabled = parseBool(val)
-	case "natlease":
-		config.Options.NATLeaseM = parseUint(val)
-	case "natrenew":
-		config.Options.NATRenewalM = parseUint(val)
-	case "reporting":
-		switch strings.ToLower(val) {
-		case "u", "undecided", "unset":
-			config.Options.URAccepted = 0
-		default:
-			boolvalue := parseBool(val)
-			if boolvalue {
-				config.Options.URAccepted = 1
-			} else {
-				config.Options.URAccepted = -1
-			}
-		}
-	case "wake":
-		config.Options.RestartOnWakeup = parseBool(val)
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
-	}
-	setConfig(c, config)
-}

cmd/stcli/cmd_report.go

@@ -1,72 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "report",
-		HideHelp: true,
-		Usage:    "Reporting command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "system",
-				Usage:    "Report system state",
-				Requires: &cli.Requires{},
-				Action:   reportSystem,
-			},
-			{
-				Name:     "connections",
-				Usage:    "Report about connections to other devices",
-				Requires: &cli.Requires{},
-				Action:   reportConnections,
-			},
-			{
-				Name:     "usage",
-				Usage:    "Usage report",
-				Requires: &cli.Requires{},
-				Action:   reportUsage,
-			},
-		},
-	})
-}
-
-func reportSystem(c *cli.Context) {
-	response := httpGet(c, "system/status")
-	data := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &data)
-	prettyPrintJSON(data)
-}
-
-func reportConnections(c *cli.Context) {
-	response := httpGet(c, "system/connections")
-	data := make(map[string]map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &data)
-	var overall map[string]interface{}
-	for key, value := range data {
-		if key == "total" {
-			overall = value
-			continue
-		}
-		value["Device ID"] = key
-		prettyPrintJSON(value)
-		fmt.Println()
-	}
-	if overall != nil {
-		fmt.Println("=== Overall statistics ===")
-		prettyPrintJSON(overall)
-	}
-}
-
-func reportUsage(c *cli.Context) {
-	response := httpGet(c, "svc/report")
-	report := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &report)
-	prettyPrintJSON(report)
-}

cmd/stcli/errors.go

@@ -0,0 +1,61 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/urfave/cli"
+)
+
+var errorsCommand = cli.Command{
+	Name:     "errors",
+	HideHelp: true,
+	Usage:    "Error command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "show",
+			Usage:  "Show pending errors",
+			Action: expects(0, dumpOutput("system/error")),
+		},
+		{
+			Name:      "push",
+			Usage:     "Push an error to active clients",
+			ArgsUsage: "[error message]",
+			Action:    expects(1, errorsPush),
+		},
+		{
+			Name:   "clear",
+			Usage:  "Clear pending errors",
+			Action: expects(0, emptyPost("system/error/clear")),
+		},
+	},
+}
+
+func errorsPush(c *cli.Context) error {
+	client := c.App.Metadata["client"].(*APIClient)
+	errStr := strings.Join(c.Args(), " ")
+	response, err := client.Post("system/error", strings.TrimSpace(errStr))
+	if err != nil {
+		return err
+	}
+	if response.StatusCode != 200 {
+		errStr = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
+		bytes, err := responseToBArray(response)
+		if err != nil {
+			return err
+		}
+		body := string(bytes)
+		if body != "" {
+			errStr += "\nBody: " + body
+		}
+		return errors.New(errStr)
+	}
+	return nil
+}

cmd/stcli/labels.go

@@ -1,31 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-var jsonAttributeLabels = map[string]string{
-	"folderMaxMiB":   "Largest folder size in MiB",
-	"folderMaxFiles": "Largest folder file count",
-	"longVersion":    "Long version",
-	"totMiB":         "Total size in MiB",
-	"totFiles":       "Total files",
-	"uniqueID":       "Unique ID",
-	"numFolders":     "Folder count",
-	"numDevices":     "Device count",
-	"memoryUsageMiB": "Memory usage in MiB",
-	"memorySize":     "Total memory in MiB",
-	"sha256Perf":     "SHA256 Benchmark",
-	"At":             "Last contacted",
-	"Completion":     "Percent complete",
-	"InBytesTotal":   "Total bytes received",
-	"OutBytesTotal":  "Total bytes sent",
-	"ClientVersion":  "Client version",
-	"alloc":          "Memory allocated in bytes",
-	"sys":            "Memory using in bytes",
-	"cpuPercent":     "CPU load in percent",
-	"extAnnounceOK":  "External announcments working",
-	"goroutines":     "Number of Go routines",
-	"myID":           "Client ID",
-	"tilde":          "Tilde expands to",
-	"arch":           "Architecture",
-	"os":             "OS",
-}

cmd/stcli/main.go

@@ -1,63 +1,192 @@
-// Copyright (C) 2014 Audrius Butkevičius
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
-	"sort"
+	"bufio"
+	"crypto/tls"
+	"encoding/json"
+	"flag"
+	"log"
+	"os"
+	"reflect"
 
-	"github.com/AudriusButkevicius/cli"
+	"github.com/AudriusButkevicius/recli"
+	"github.com/flynn-archive/go-shlex"
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/locations"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/urfave/cli"
 )
 
-type ByAlphabet []cli.Command
-
-func (a ByAlphabet) Len() int           { return len(a) }
-func (a ByAlphabet) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
-func (a ByAlphabet) Less(i, j int) bool { return a[i].Name < a[j].Name }
-
-var cliCommands []cli.Command
-
 func main() {
-	app := cli.NewApp()
+	// This is somewhat a hack around a chicken and egg problem.
-	app.Name = "syncthing-cli"
+	// We need to set the home directory and potentially other flags to know where the syncthing instance is running
-	app.Author = "Audrius Butkevičius"
+	// in order to get it's config ... which we then use to construct the actual CLI ... at which point it's too late
-	app.Email = "audrius.butkevicius@gmail.com"
+	// to add flags there...
-	app.Usage = "Syncthing command line interface"
+	homeBaseDir := locations.GetBaseDir(locations.ConfigBaseDir)
-	app.Version = "0.1"
+	guiCfg := config.GUIConfiguration{}
-	app.HideHelp = true
 
-	app.Flags = []cli.Flag{
+	flags := flag.NewFlagSet("", flag.ContinueOnError)
+	flags.StringVar(&guiCfg.RawAddress, "gui-address", guiCfg.RawAddress, "Override GUI address (e.g. \"http://192.0.2.42:8443\")")
+	flags.StringVar(&guiCfg.APIKey, "gui-apikey", guiCfg.APIKey, "Override GUI API key")
+	flags.StringVar(&homeBaseDir, "home", homeBaseDir, "Set configuration directory")
+
+	// Implement the same flags at the lower CLI, with the same default values (pre-parse), but do nothing with them.
+	// This is so that we could reuse os.Args
+	fakeFlags := []cli.Flag{
 		cli.StringFlag{
-			Name:   "endpoint, e",
+			Name:  "gui-address",
-			Value:  "http://127.0.0.1:8384",
+			Value: guiCfg.RawAddress,
-			Usage:  "End point to connect to",
+			Usage: "Override GUI address (e.g. \"http://192.0.2.42:8443\")",
-			EnvVar: "STENDPOINT",
 		},
 		cli.StringFlag{
-			Name:   "apikey, k",
+			Name:  "gui-apikey",
-			Value:  "",
+			Value: guiCfg.APIKey,
-			Usage:  "API Key",
+			Usage: "Override GUI API key",
-			EnvVar: "STAPIKEY",
 		},
 		cli.StringFlag{
-			Name:   "username, u",
+			Name:  "home",
-			Value:  "",
+			Value: homeBaseDir,
-			Usage:  "Username",
+			Usage: "Set configuration directory",
-			EnvVar: "STUSERNAME",
-		},
-		cli.StringFlag{
-			Name:   "password, p",
-			Value:  "",
-			Usage:  "Password",
-			EnvVar: "STPASSWORD",
-		},
-		cli.BoolFlag{
-			Name:   "insecure, i",
-			Usage:  "Do not verify SSL certificate",
-			EnvVar: "STINSECURE",
 		},
 	}
 
-	sort.Sort(ByAlphabet(cliCommands))
+	// Do not print usage of these flags, and ignore errors as this can't understand plenty of things
-	app.Commands = cliCommands
+	flags.Usage = func() {}
-	app.RunAndExitOnError()
+	_ = flags.Parse(os.Args[1:])
+
+	// Now if the API key and address is not provided (we are not connecting to a remote instance),
+	// try to rip it out of the config.
+	if guiCfg.RawAddress == "" && guiCfg.APIKey == "" {
+		// Update the base directory
+		err := locations.SetBaseDir(locations.ConfigBaseDir, homeBaseDir)
+		if err != nil {
+			log.Fatal(errors.Wrap(err, "setting home"))
+		}
+
+		// Load the certs and get the ID
+		cert, err := tls.LoadX509KeyPair(
+			locations.Get(locations.CertFile),
+			locations.Get(locations.KeyFile),
+		)
+		if err != nil {
+			log.Fatal(errors.Wrap(err, "reading device ID"))
+		}
+
+		myID := protocol.NewDeviceID(cert.Certificate[0])
+
+		// Load the config
+		cfg, err := config.Load(locations.Get(locations.ConfigFile), myID, events.NoopLogger)
+		if err != nil {
+			log.Fatalln(errors.Wrap(err, "loading config"))
+		}
+
+		guiCfg = cfg.GUI()
+	} else if guiCfg.Address() == "" || guiCfg.APIKey == "" {
+		log.Fatalln("Both -gui-address and -gui-apikey should be specified")
+	}
+
+	if guiCfg.Address() == "" {
+		log.Fatalln("Could not find GUI Address")
+	}
+
+	if guiCfg.APIKey == "" {
+		log.Fatalln("Could not find GUI API key")
+	}
+
+	client := getClient(guiCfg)
+
+	cfg, err := getConfig(client)
+	original := cfg.Copy()
+	if err != nil {
+		log.Fatalln(errors.Wrap(err, "getting config"))
+	}
+
+	// Copy the config and set the default flags
+	recliCfg := recli.DefaultConfig
+	recliCfg.IDTag.Name = "xml"
+	recliCfg.SkipTag.Name = "json"
+
+	commands, err := recli.New(recliCfg).Construct(&cfg)
+	if err != nil {
+		log.Fatalln(errors.Wrap(err, "config reflect"))
+	}
+
+	// Construct the actual CLI
+	app := cli.NewApp()
+	app.Name = "stcli"
+	app.HelpName = app.Name
+	app.Author = "The Syncthing Authors"
+	app.Usage = "Syncthing command line interface"
+	app.Version = build.Version
+	app.Flags = fakeFlags
+	app.Metadata = map[string]interface{}{
+		"client": client,
+	}
+	app.Commands = []cli.Command{
+		{
+			Name:        "config",
+			HideHelp:    true,
+			Usage:       "Configuration modification command group",
+			Subcommands: commands,
+		},
+		showCommand,
+		operationCommand,
+		errorsCommand,
+	}
+
+	tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
+	if !tty {
+		// Not a TTY, consume from stdin
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			input, err := shlex.Split(scanner.Text())
+			if err != nil {
+				log.Fatalln(errors.Wrap(err, "parsing input"))
+			}
+			if len(input) == 0 {
+				continue
+			}
+			err = app.Run(append(os.Args, input...))
+			if err != nil {
+				log.Fatalln(err)
+			}
+		}
+		err = scanner.Err()
+		if err != nil {
+			log.Fatalln(err)
+		}
+	} else {
+		err = app.Run(os.Args)
+		if err != nil {
+			log.Fatalln(err)
+		}
+	}
+
+	if !reflect.DeepEqual(cfg, original) {
+		body, err := json.MarshalIndent(cfg, "", "  ")
+		if err != nil {
+			log.Fatalln(err)
+		}
+		resp, err := client.Post("system/config", string(body))
+		if err != nil {
+			log.Fatalln(err)
+		}
+		if resp.StatusCode != 200 {
+			body, err := responseToBArray(resp)
+			if err != nil {
+				log.Fatalln(err)
+			}
+			log.Fatalln(string(body))
+		}
+	}
 }

cmd/stcli/operations.go

@@ -0,0 +1,78 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli"
+)
+
+var operationCommand = cli.Command{
+	Name:     "operations",
+	HideHelp: true,
+	Usage:    "Operation command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "restart",
+			Usage:  "Restart syncthing",
+			Action: expects(0, emptyPost("system/restart")),
+		},
+		{
+			Name:   "shutdown",
+			Usage:  "Shutdown syncthing",
+			Action: expects(0, emptyPost("system/shutdown")),
+		},
+		{
+			Name:   "reset",
+			Usage:  "Reset syncthing deleting all folders and devices",
+			Action: expects(0, emptyPost("system/reset")),
+		},
+		{
+			Name:   "upgrade",
+			Usage:  "Upgrade syncthing (if a newer version is available)",
+			Action: expects(0, emptyPost("system/upgrade")),
+		},
+		{
+			Name:      "folder-override",
+			Usage:     "Override changes on folder (remote for sendonly, local for receiveonly)",
+			ArgsUsage: "[folder id]",
+			Action:    expects(1, foldersOverride),
+		},
+	},
+}
+
+func foldersOverride(c *cli.Context) error {
+	client := c.App.Metadata["client"].(*APIClient)
+	cfg, err := getConfig(client)
+	if err != nil {
+		return err
+	}
+	rid := c.Args()[0]
+	for _, folder := range cfg.Folders {
+		if folder.ID == rid {
+			response, err := client.Post("db/override", "")
+			if err != nil {
+				return err
+			}
+			if response.StatusCode != 200 {
+				errStr := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
+				bytes, err := responseToBArray(response)
+				if err != nil {
+					return err
+				}
+				body := string(bytes)
+				if body != "" {
+					errStr += "\nBody: " + body
+				}
+				return fmt.Errorf(errStr)
+			}
+			return nil
+		}
+	}
+	return fmt.Errorf("Folder " + rid + " not found")
+}

cmd/stcli/show.go

@@ -0,0 +1,44 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"github.com/urfave/cli"
+)
+
+var showCommand = cli.Command{
+	Name:     "show",
+	HideHelp: true,
+	Usage:    "Show command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "version",
+			Usage:  "Show syncthing client version",
+			Action: expects(0, dumpOutput("system/version")),
+		},
+		{
+			Name:   "config-status",
+			Usage:  "Show configuration status, whether or not a restart is required for changes to take effect",
+			Action: expects(0, dumpOutput("system/config/insync")),
+		},
+		{
+			Name:   "system",
+			Usage:  "Show system status",
+			Action: expects(0, dumpOutput("system/status")),
+		},
+		{
+			Name:   "connections",
+			Usage:  "Report about connections to other devices",
+			Action: expects(0, dumpOutput("system/connections")),
+		},
+		{
+			Name:   "usage",
+			Usage:  "Show usage report",
+			Action: expects(0, dumpOutput("svc/report")),
+		},
+	},
+}

cmd/stcli/utils.go

@@ -1,4 +1,8 @@
-// Copyright (C) 2014 Audrius Butkevičius
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -8,158 +12,83 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"text/tabwriter"
-	"unicode"
 
-	"github.com/AudriusButkevicius/cli"
 	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/urfave/cli"
 )
 
-func responseToBArray(response *http.Response) []byte {
+func responseToBArray(response *http.Response) ([]byte, error) {
-	defer response.Body.Close()
 	bytes, err := ioutil.ReadAll(response.Body)
 	if err != nil {
-		die(err)
+		return nil, err
 	}
-	return bytes
+	return bytes, response.Body.Close()
 }
 
-func die(vals ...interface{}) {
+func emptyPost(url string) cli.ActionFunc {
-	if len(vals) > 1 || vals[0] != nil {
+	return func(c *cli.Context) error {
-		os.Stderr.WriteString(fmt.Sprintln(vals...))
+		client := c.App.Metadata["client"].(*APIClient)
-		os.Exit(1)
+		_, err := client.Post(url, "")
+		return err
 	}
 }
 
-func wrappedHTTPPost(url string) func(c *cli.Context) {
+func dumpOutput(url string) cli.ActionFunc {
-	return func(c *cli.Context) {
+	return func(c *cli.Context) error {
-		httpPost(c, url, "")
+		client := c.App.Metadata["client"].(*APIClient)
-	}
+		response, err := client.Get(url)
-}
+		if err != nil {
-
+			return err
-func prettyPrintJSON(json map[string]interface{}) {
-	writer := newTableWriter()
-	remap := make(map[string]interface{})
-	for k, v := range json {
-		key, ok := jsonAttributeLabels[k]
-		if !ok {
-			key = firstUpper(k)
 		}
-		remap[key] = v
+		return prettyPrintResponse(c, response)
 	}
+}
 
-	jsonKeys := make([]string, 0, len(remap))
+func getConfig(c *APIClient) (config.Configuration, error) {
-	for key := range remap {
+	cfg := config.Configuration{}
-		jsonKeys = append(jsonKeys, key)
+	response, err := c.Get("system/config")
+	if err != nil {
+		return cfg, err
 	}
-	sort.Strings(jsonKeys)
+	bytes, err := responseToBArray(response)
-	for _, k := range jsonKeys {
+	if err != nil {
-		var value string
+		return cfg, err
-		rvalue := remap[k]
+	}
-		switch rvalue.(type) {
+	err = json.Unmarshal(bytes, &cfg)
-		case int, int16, int32, int64, uint, uint16, uint32, uint64, float32, float64:
+	if err == nil {
-			value = fmt.Sprintf("%.0f", rvalue)
+		return cfg, err
-		default:
+	}
-			value = fmt.Sprint(rvalue)
+	return cfg, nil
+}
+
+func expects(n int, actionFunc cli.ActionFunc) cli.ActionFunc {
+	return func(ctx *cli.Context) error {
+		if ctx.NArg() != n {
+			plural := ""
+			if n != 1 {
+				plural = "s"
+			}
+			return fmt.Errorf("expected %d argument%s, got %d", n, plural, ctx.NArg())
 		}
-		if value == "" {
+		return actionFunc(ctx)
-			continue
-		}
-		fmt.Fprintln(writer, k+":\t"+value)
-	}
-	writer.Flush()
-}
-
-func firstUpper(str string) string {
-	for i, v := range str {
-		return string(unicode.ToUpper(v)) + str[i+1:]
-	}
-	return ""
-}
-
-func newTableWriter() *tabwriter.Writer {
-	writer := new(tabwriter.Writer)
-	writer.Init(os.Stdout, 0, 8, 0, '\t', 0)
-	return writer
-}
-
-func getMyID(c *cli.Context) string {
-	response := httpGet(c, "system/status")
-	data := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &data)
-	return data["myID"].(string)
-}
-
-func getConfig(c *cli.Context) config.Configuration {
-	response := httpGet(c, "system/config")
-	config := config.Configuration{}
-	json.Unmarshal(responseToBArray(response), &config)
-	return config
-}
-
-func setConfig(c *cli.Context, cfg config.Configuration) {
-	body, err := json.Marshal(cfg)
-	die(err)
-	response := httpPost(c, "system/config", string(body))
-	if response.StatusCode != 200 {
-		die("Unexpected status code", response.StatusCode)
 	}
 }
 
-func parseBool(input string) bool {
+func prettyPrintJSON(data interface{}) error {
-	val, err := strconv.ParseBool(input)
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	return enc.Encode(data)
+}
+
+func prettyPrintResponse(c *cli.Context, response *http.Response) error {
+	bytes, err := responseToBArray(response)
 	if err != nil {
-		die(input + " is not a valid value for a boolean")
+		return err
 	}
-	return val
+	var data interface{}
-}
+	if err := json.Unmarshal(bytes, &data); err != nil {
-
+		return err
-func parseInt(input string) int {
+	}
-	val, err := strconv.ParseInt(input, 0, 64)
+	// TODO: Check flag for pretty print format
-	if err != nil {
+	return prettyPrintJSON(data)
-		die(input + " is not a valid value for an integer")
-	}
-	return int(val)
-}
-
-func parseUint(input string) int {
-	val, err := strconv.ParseUint(input, 0, 64)
-	if err != nil {
-		die(input + " is not a valid value for an unsigned integer")
-	}
-	return int(val)
-}
-
-func parsePort(input string) int {
-	port := parseUint(input)
-	if port < 1 || port > 65535 {
-		die(input + " is not a valid port\nExpected value between 1 and 65535")
-	}
-	return port
-}
-
-func validAddress(input string) {
-	tokens := strings.Split(input, ":")
-	if len(tokens) != 2 {
-		die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
-	}
-	matched, err := regexp.MatchString("^[a-zA-Z0-9]+([-a-zA-Z0-9.]+[-a-zA-Z0-9]+)?$", tokens[0])
-	die(err)
-	if !matched {
-		die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
-	}
-	parsePort(tokens[1])
-}
-
-func parseDeviceID(input string) protocol.DeviceID {
-	device, err := protocol.DeviceIDFromString(input)
-	if err != nil {
-		die(input + " is not a valid device id")
-	}
-	return device
 }

cmd/stcompdirs/main.go

@@ -60,7 +60,7 @@ func compareDirectories(dirs ...string) error {
 				} else if res[i].name > res[0].name {
 					return fmt.Errorf("%s missing %v (present in %s)", dirs[i], res[0], dirs[0])
 				}
-				return fmt.Errorf("Mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
+				return fmt.Errorf("mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
 			}
 		}
 

cmd/stcrashreceiver/sentry.go

@@ -0,0 +1,206 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"regexp"
+	"strings"
+	"sync"
+
+	raven "github.com/getsentry/raven-go"
+	"github.com/maruel/panicparse/stack"
+)
+
+const reportServer = "https://crash.syncthing.net/report/"
+
+var loader = newGithubSourceCodeLoader()
+
+func init() {
+	raven.SetSourceCodeLoader(loader)
+}
+
+var (
+	clients    = make(map[string]*raven.Client)
+	clientsMut sync.Mutex
+)
+
+func sendReport(dsn, path string, report []byte, userID string) error {
+	pkt, err := parseReport(path, report)
+	if err != nil {
+		return err
+	}
+
+	pkt.Interfaces = append(pkt.Interfaces, &raven.User{ID: userID})
+
+	clientsMut.Lock()
+	defer clientsMut.Unlock()
+
+	cli, ok := clients[dsn]
+	if !ok {
+		cli, err = raven.New(dsn)
+		if err != nil {
+			return err
+		}
+		clients[dsn] = cli
+	}
+
+	// The client sets release and such on the packet before sending, in the
+	// misguided idea that it knows this better than than the packet we give
+	// it. So we copy the values from the packet to the client first...
+	cli.SetRelease(pkt.Release)
+	cli.SetEnvironment(pkt.Environment)
+
+	defer cli.Wait()
+	_, errC := cli.Capture(pkt, nil)
+	return <-errC
+}
+
+func parseReport(path string, report []byte) (*raven.Packet, error) {
+	parts := bytes.SplitN(report, []byte("\n"), 2)
+	if len(parts) != 2 {
+		return nil, errors.New("no first line")
+	}
+
+	version, err := parseVersion(string(parts[0]))
+	if err != nil {
+		return nil, err
+	}
+	report = parts[1]
+
+	foundPanic := false
+	var subjectLine []byte
+	for {
+		parts = bytes.SplitN(report, []byte("\n"), 2)
+		if len(parts) != 2 {
+			return nil, errors.New("no panic line found")
+		}
+
+		line := parts[0]
+		report = parts[1]
+
+		if foundPanic {
+			// The previous line was our "Panic at ..." header. We are now
+			// at the beginning of the real panic trace and this is our
+			// subject line.
+			subjectLine = line
+			break
+		} else if bytes.HasPrefix(line, []byte("Panic at")) {
+			foundPanic = true
+		}
+	}
+
+	r := bytes.NewReader(report)
+	ctx, err := stack.ParseDump(r, ioutil.Discard, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// Lock the source code loader to the version we are processing here.
+	if version.commit != "" {
+		// We have a commit hash, so we know exactly which source to use
+		loader.LockWithVersion(version.commit)
+	} else if strings.HasPrefix(version.tag, "v") {
+		// Lets hope the tag is close enough
+		loader.LockWithVersion(version.tag)
+	} else {
+		// Last resort
+		loader.LockWithVersion("master")
+	}
+	defer loader.Unlock()
+
+	var trace raven.Stacktrace
+	for _, gr := range ctx.Goroutines {
+		if gr.First {
+			trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
+			for i, sc := range gr.Stack.Calls {
+				trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name(), sc.SrcPath, sc.Line, 3, nil)
+			}
+			break
+		}
+	}
+
+	pkt := &raven.Packet{
+		Message:     string(subjectLine),
+		Platform:    "go",
+		Release:     version.tag,
+		Environment: version.environment(),
+		Tags: raven.Tags{
+			raven.Tag{Key: "version", Value: version.version},
+			raven.Tag{Key: "tag", Value: version.tag},
+			raven.Tag{Key: "codename", Value: version.codename},
+			raven.Tag{Key: "runtime", Value: version.runtime},
+			raven.Tag{Key: "goos", Value: version.goos},
+			raven.Tag{Key: "goarch", Value: version.goarch},
+			raven.Tag{Key: "builder", Value: version.builder},
+		},
+		Extra: raven.Extra{
+			"url": reportServer + path,
+		},
+		Interfaces: []raven.Interface{&trace},
+	}
+	if version.commit != "" {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
+	}
+
+	return pkt, nil
+}
+
+// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC
+var longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)`)
+
+type version struct {
+	version  string // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
+	tag      string // "v1.1.4-rc.1"
+	commit   string // "6aaae618", blank when absent
+	codename string // "Erbium Earthworm"
+	runtime  string // "go1.12.5"
+	goos     string // "darwin"
+	goarch   string // "amd64"
+	builder  string // "jb@kvin.kastelo.net"
+}
+
+func (v version) environment() string {
+	if v.commit != "" {
+		return "Development"
+	}
+	if strings.Contains(v.tag, "-rc.") {
+		return "Candidate"
+	}
+	if strings.Contains(v.tag, "-") {
+		return "Beta"
+	}
+	return "Stable"
+}
+
+func parseVersion(line string) (version, error) {
+	m := longVersionRE.FindStringSubmatch(line)
+	if len(m) == 0 {
+		return version{}, errors.New("unintelligeble version string")
+	}
+
+	v := version{
+		version:  m[1],
+		codename: m[2],
+		runtime:  m[3],
+		goos:     m[4],
+		goarch:   m[5],
+		builder:  m[6],
+	}
+	parts := strings.Split(v.version, "+")
+	v.tag = parts[0]
+	if len(parts) > 1 {
+		fields := strings.Split(parts[1], "-")
+		if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
+			v.commit = fields[1][1:]
+		}
+	}
+
+	return v, nil
+}

cmd/stcrashreceiver/sentry_test.go

@@ -0,0 +1,64 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+)
+
+func TestParseVersion(t *testing.T) {
+	cases := []struct {
+		longVersion string
+		parsed      version
+	}{
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		v, err := parseVersion(tc.longVersion)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		if v != tc.parsed {
+			t.Error(v)
+		}
+	}
+}
+
+func TestParseReport(t *testing.T) {
+	bs, err := ioutil.ReadFile("_testdata/panic.log")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pkt, err := parseReport("1/2/345", bs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bs, err = pkt.JSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Printf("%s\n", bs)
+}

cmd/stcrashreceiver/sourcecodeloader.go

@@ -0,0 +1,114 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	urlPrefix   = "https://raw.githubusercontent.com/syncthing/syncthing/"
+	httpTimeout = 10 * time.Second
+)
+
+type githubSourceCodeLoader struct {
+	mut     sync.Mutex
+	version string
+	cache   map[string]map[string][][]byte // version -> file -> lines
+	client  *http.Client
+}
+
+func newGithubSourceCodeLoader() *githubSourceCodeLoader {
+	return &githubSourceCodeLoader{
+		cache:  make(map[string]map[string][][]byte),
+		client: &http.Client{Timeout: httpTimeout},
+	}
+}
+
+func (l *githubSourceCodeLoader) LockWithVersion(version string) {
+	l.mut.Lock()
+	l.version = version
+	if _, ok := l.cache[version]; !ok {
+		l.cache[version] = make(map[string][][]byte)
+	}
+}
+
+func (l *githubSourceCodeLoader) Unlock() {
+	l.mut.Unlock()
+}
+
+func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]byte, int) {
+	filename = filepath.ToSlash(filename)
+	lines, ok := l.cache[l.version][filename]
+	if !ok {
+		// Cache whatever we managed to find (or nil if nothing, so we don't try again)
+		defer func() {
+			l.cache[l.version][filename] = lines
+		}()
+
+		knownPrefixes := []string{"/lib/", "/cmd/"}
+		var idx int
+		for _, pref := range knownPrefixes {
+			idx = strings.Index(filename, pref)
+			if idx >= 0 {
+				break
+			}
+		}
+		if idx == -1 {
+			return nil, 0
+		}
+
+		url := urlPrefix + l.version + filename[idx:]
+		resp, err := l.client.Get(url)
+
+		if err != nil || resp.StatusCode != http.StatusOK {
+			fmt.Println("Loading source:", err.Error())
+			return nil, 0
+		}
+		data, err := ioutil.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		if err != nil {
+			fmt.Println("Loading source:", err.Error())
+			return nil, 0
+		}
+		lines = bytes.Split(data, []byte{'\n'})
+	}
+
+	return getLineFromLines(lines, line, context)
+}
+
+func getLineFromLines(lines [][]byte, line, context int) ([][]byte, int) {
+	if lines == nil {
+		// cached error from ReadFile: return no lines
+		return nil, 0
+	}
+
+	line-- // stack trace lines are 1-indexed
+	start := line - context
+	var idx int
+	if start < 0 {
+		start = 0
+		idx = line
+	} else {
+		idx = context
+	}
+	end := line + context + 1
+	if line >= len(lines) {
+		return nil, 0
+	}
+	if end > len(lines) {
+		end = len(lines)
+	}
+	return lines[start:end], idx
+}

cmd/stcrashreceiver/stcrashreceiver.go

@@ -0,0 +1,185 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Command stcrashreceiver is a trivial HTTP server that allows two things:
+//
+// - uploading files (crash reports) named like a SHA256 hash using a PUT request
+// - checking whether such file exists using a HEAD request
+//
+// Typically this should be deployed behind something that manages HTTPS.
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/sha256"
+)
+
+const maxRequestSize = 1 << 20 // 1 MiB
+
+func main() {
+	dir := flag.String("dir", ".", "Directory to store reports in")
+	dsn := flag.String("dsn", "", "Sentry DSN")
+	listen := flag.String("listen", ":22039", "HTTP listen address")
+	flag.Parse()
+
+	cr := &crashReceiver{
+		dir: *dir,
+		dsn: *dsn,
+	}
+
+	log.SetOutput(os.Stdout)
+	if err := http.ListenAndServe(*listen, cr); err != nil {
+		log.Fatalln("HTTP serve:", err)
+	}
+}
+
+type crashReceiver struct {
+	dir string
+	dsn string
+}
+
+func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	// The final path component should be a SHA256 hash in hex, so 64 hex
+	// characters. We don't care about case on the request but use lower
+	// case internally.
+	reportID := strings.ToLower(path.Base(req.URL.Path))
+	if len(reportID) != 64 {
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+	for _, c := range reportID {
+		if c >= 'a' && c <= 'f' {
+			continue
+		}
+		if c >= '0' && c <= '9' {
+			continue
+		}
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+
+	// The location of the report on disk, compressed
+	fullPath := filepath.Join(r.dir, r.dirFor(reportID), reportID) + ".gz"
+
+	switch req.Method {
+	case http.MethodGet:
+		r.serveGet(fullPath, w, req)
+	case http.MethodHead:
+		r.serveHead(fullPath, w, req)
+	case http.MethodPut:
+		r.servePut(reportID, fullPath, w, req)
+	default:
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+// serveGet responds to GET requests by serving the uncompressed report.
+func (r *crashReceiver) serveGet(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	fd, err := os.Open(fullPath)
+	if err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+		return
+	}
+
+	defer fd.Close()
+	gr, err := gzip.NewReader(fd)
+	if err != nil {
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+	_, _ = io.Copy(w, gr) // best effort
+}
+
+// serveHead responds to HEAD requests by checking if the named report
+// already exists in the system.
+func (r *crashReceiver) serveHead(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	if _, err := os.Lstat(fullPath); err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+	}
+}
+
+// servePut accepts and stores the given report.
+func (r *crashReceiver) servePut(reportID, fullPath string, w http.ResponseWriter, req *http.Request) {
+	// Ensure the destination directory exists
+	if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
+		log.Println("Creating directory:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Read at most maxRequestSize of report data.
+	log.Println("Receiving report", reportID)
+	lr := io.LimitReader(req.Body, maxRequestSize)
+	bs, err := ioutil.ReadAll(lr)
+	if err != nil {
+		log.Println("Reading report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Compress the report for storage
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	_, _ = gw.Write(bs) // can't fail
+	gw.Close()
+
+	// Create an output file with the compressed report
+	err = ioutil.WriteFile(fullPath, buf.Bytes(), 0644)
+	if err != nil {
+		log.Println("Saving report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Send the report to Sentry
+	if r.dsn != "" {
+		// Remote ID
+		user := userIDFor(req)
+
+		go func() {
+			// There's no need for the client to have to wait for this part.
+			if err := sendReport(r.dsn, reportID, bs, user); err != nil {
+				log.Println("Failed to send report:", err)
+			}
+		}()
+	}
+}
+
+// 01234567890abcdef... => 01/23
+func (r *crashReceiver) dirFor(base string) string {
+	return filepath.Join(base[0:2], base[2:4])
+}
+
+// userIDFor returns a string we can use as the user ID for the purpose of
+// counting affected users. It's the truncated hash of a salt, the user
+// remote IP, and the current month.
+func userIDFor(req *http.Request) string {
+	addr := req.RemoteAddr
+	if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
+		addr = fwd
+	}
+	if host, _, err := net.SplitHostPort(addr); err == nil {
+		addr = host
+	}
+	now := time.Now().Format("200601")
+	salt := "stcrashreporter"
+	hash := sha256.Sum256([]byte(salt + addr + now))
+	return fmt.Sprintf("%x", hash[:8])
+}

cmd/stdiscosrv/apisrv.go

@@ -18,6 +18,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -279,6 +280,10 @@ func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addre
 		dbAddrs[i].Expires = expire
 	}
 
+	// The address slice must always be sorted for database merges to work
+	// properly.
+	sort.Sort(databaseAddressOrder(dbAddrs))
+
 	seen := now.UnixNano()
 	if s.repl != nil {
 		s.repl.send(key, dbAddrs, seen)
@@ -295,28 +300,66 @@ func certificateBytes(req *http.Request) []byte {
 		return req.TLS.PeerCertificates[0].Raw
 	}
 
+	var bs []byte
+
 	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
-		bs := []byte(hdr)
+		if strings.Contains(hdr, "%") {
-		// The certificate is in PEM format but with spaces for newlines. We
+			// Nginx using $ssl_client_escaped_cert
-		// need to reinstate the newlines for the PEM decoder. But we need to
+			// The certificate is in PEM format with url encoding.
-		// leave the spaces in the BEGIN and END lines - the first and last
+			// We need to decode for the PEM decoder
-		// space - alone.
+			hdr, err := url.QueryUnescape(hdr)
-		firstSpace := bytes.Index(bs, []byte(" "))
+			if err != nil {
-		lastSpace := bytes.LastIndex(bs, []byte(" "))
+				// Decoding failed
-		for i := firstSpace + 1; i < lastSpace; i++ {
+				return nil
-			if bs[i] == ' ' {
+			}
-				bs[i] = '\n'
+
+			bs = []byte(hdr)
+		} else {
+			// Nginx using $ssl_client_cert
+			// The certificate is in PEM format but with spaces for newlines. We
+			// need to reinstate the newlines for the PEM decoder. But we need to
+			// leave the spaces in the BEGIN and END lines - the first and last
+			// space - alone.
+			bs = []byte(hdr)
+			firstSpace := bytes.Index(bs, []byte(" "))
+			lastSpace := bytes.LastIndex(bs, []byte(" "))
+			for i := firstSpace + 1; i < lastSpace; i++ {
+				if bs[i] == ' ' {
+					bs[i] = '\n'
+				}
 			}
 		}
-		block, _ := pem.Decode(bs)
+	} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
-		if block == nil {
+		// Traefik 2 passtlsclientcert
+		// The certificate is in PEM format with url encoding but without newlines
+		// and start/end statements. We need to decode, reinstate the newlines every 64
+		// character and add statements for the PEM decoder
+		hdr, err := url.QueryUnescape(hdr)
+		if err != nil {
 			// Decoding failed
 			return nil
 		}
-		return block.Bytes
+
+		for i := 64; i < len(hdr); i += 65 {
+			hdr = hdr[:i] + "\n" + hdr[i:]
+		}
+
+		hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
+		hdr = hdr + "\n-----END CERTIFICATE-----\n"
+		bs = []byte(hdr)
 	}
 
-	return nil
+	if bs == nil {
+		return nil
+	}
+
+	block, _ := pem.Decode(bs)
+	if block == nil {
+		// Decoding failed
+		return nil
+	}
+
+	return block.Bytes
 }
 
 // fixupAddresses checks the list of addresses, removing invalid ones and

cmd/stdiscosrv/database.go

@@ -5,11 +5,12 @@
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
 //go:generate go run ../../script/protofmt.go database.proto
-//go:generate protoc -I ../../../../../ -I ../../vendor/ -I ../../vendor/github.com/gogo/protobuf/protobuf -I . --gogofast_out=. database.proto
+//go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
 
 package main
 
 import (
+	"log"
 	"sort"
 	"time"
 
@@ -162,10 +163,10 @@ func (s *levelDBStore) Serve() {
 	// Start the statistics serve routine. It will exit with us when
 	// statisticsTrigger is closed.
 	statisticsTrigger := make(chan struct{})
-	defer close(statisticsTrigger)
 	statisticsDone := make(chan struct{})
 	go s.statisticsServe(statisticsTrigger, statisticsDone)
 
+loop:
 	for {
 		select {
 		case fn := <-s.inbox:
@@ -184,12 +185,18 @@ func (s *levelDBStore) Serve() {
 
 		case <-s.stop:
 			// We're done.
-			return
+			close(statisticsTrigger)
+			break loop
 		}
 	}
+
+	// Also wait for statisticsServe to return
+	<-statisticsDone
 }
 
 func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
+	defer close(done)
+
 	for range trigger {
 		t0 := time.Now()
 		nowNanos := t0.UnixNano()
@@ -257,12 +264,15 @@ func (s *levelDBStore) Stop() {
 // chosen for any duplicates.
 func merge(a, b DatabaseRecord) DatabaseRecord {
 	// Both lists must be sorted for this to work.
-	sort.Slice(a.Addresses, func(i, j int) bool {
+	if !sort.IsSorted(databaseAddressOrder(a.Addresses)) {
-		return a.Addresses[i].Address < a.Addresses[j].Address
+		log.Println("Warning: bug: addresses not correctly sorted in merge")
-	})
+		a.Addresses = sortedAddressCopy(a.Addresses)
-	sort.Slice(b.Addresses, func(i, j int) bool {
+	}
-		return b.Addresses[i].Address < b.Addresses[j].Address
+	if !sort.IsSorted(databaseAddressOrder(b.Addresses)) {
-	})
+		// no warning because this is the side we read from disk and it may
+		// legitimately predate correct sorting.
+		b.Addresses = sortedAddressCopy(b.Addresses)
+	}
 
 	res := DatabaseRecord{
 		Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
@@ -346,3 +356,24 @@ func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
 	}
 	return addrs
 }
+
+func sortedAddressCopy(addrs []DatabaseAddress) []DatabaseAddress {
+	sorted := make([]DatabaseAddress, len(addrs))
+	copy(sorted, addrs)
+	sort.Sort(databaseAddressOrder(sorted))
+	return sorted
+}
+
+type databaseAddressOrder []DatabaseAddress
+
+func (s databaseAddressOrder) Less(a, b int) bool {
+	return s[a].Address < s[b].Address
+}
+
+func (s databaseAddressOrder) Swap(a, b int) {
+	s[a], s[b] = s[b], s[a]
+}
+
+func (s databaseAddressOrder) Len() int {
+	return len(s)
+}

cmd/stdiscosrv/database.pb.go

@@ -1,25 +1,16 @@
 // Code generated by protoc-gen-gogo. DO NOT EDIT.
 // source: database.proto
 
-/*
-	Package main is a generated protocol buffer package.
-
-	It is generated from these files:
-		database.proto
-
-	It has these top-level messages:
-		DatabaseRecord
-		ReplicationRecord
-		DatabaseAddress
-*/
 package main
 
-import proto "github.com/gogo/protobuf/proto"
+import (
-import fmt "fmt"
+	fmt "fmt"
-import math "math"
+	_ "github.com/gogo/protobuf/gogoproto"
-import _ "github.com/gogo/protobuf/gogoproto"
+	proto "github.com/gogo/protobuf/proto"
-
+	io "io"
-import io "io"
+	math "math"
+	math_bits "math/bits"
+)
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
@@ -30,50 +21,158 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
 
 type DatabaseRecord struct {
-	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses" json:"addresses"`
+	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
 	Misses    int32             `protobuf:"varint,2,opt,name=misses,proto3" json:"misses,omitempty"`
 	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
 	Missed    int64             `protobuf:"varint,4,opt,name=missed,proto3" json:"missed,omitempty"`
 }
 
-func (m *DatabaseRecord) Reset()                    { *m = DatabaseRecord{} }
+func (m *DatabaseRecord) Reset()         { *m = DatabaseRecord{} }
-func (m *DatabaseRecord) String() string            { return proto.CompactTextString(m) }
+func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
-func (*DatabaseRecord) ProtoMessage()               {}
+func (*DatabaseRecord) ProtoMessage()    {}
-func (*DatabaseRecord) Descriptor() ([]byte, []int) { return fileDescriptorDatabase, []int{0} }
+func (*DatabaseRecord) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{0}
+}
+func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseRecord.Merge(m, src)
+}
+func (m *DatabaseRecord) XXX_Size() int {
+	return m.Size()
+}
+func (m *DatabaseRecord) XXX_DiscardUnknown() {
+	xxx_messageInfo_DatabaseRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DatabaseRecord proto.InternalMessageInfo
 
 type ReplicationRecord struct {
 	Key       string            `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
-	Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses" json:"addresses"`
+	Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses"`
 	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
 }
 
-func (m *ReplicationRecord) Reset()                    { *m = ReplicationRecord{} }
+func (m *ReplicationRecord) Reset()         { *m = ReplicationRecord{} }
-func (m *ReplicationRecord) String() string            { return proto.CompactTextString(m) }
+func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
-func (*ReplicationRecord) ProtoMessage()               {}
+func (*ReplicationRecord) ProtoMessage()    {}
-func (*ReplicationRecord) Descriptor() ([]byte, []int) { return fileDescriptorDatabase, []int{1} }
+func (*ReplicationRecord) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{1}
+}
+func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ReplicationRecord.Merge(m, src)
+}
+func (m *ReplicationRecord) XXX_Size() int {
+	return m.Size()
+}
+func (m *ReplicationRecord) XXX_DiscardUnknown() {
+	xxx_messageInfo_ReplicationRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ReplicationRecord proto.InternalMessageInfo
 
 type DatabaseAddress struct {
 	Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
 	Expires int64  `protobuf:"varint,2,opt,name=expires,proto3" json:"expires,omitempty"`
 }
 
-func (m *DatabaseAddress) Reset()                    { *m = DatabaseAddress{} }
+func (m *DatabaseAddress) Reset()         { *m = DatabaseAddress{} }
-func (m *DatabaseAddress) String() string            { return proto.CompactTextString(m) }
+func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
-func (*DatabaseAddress) ProtoMessage()               {}
+func (*DatabaseAddress) ProtoMessage()    {}
-func (*DatabaseAddress) Descriptor() ([]byte, []int) { return fileDescriptorDatabase, []int{2} }
+func (*DatabaseAddress) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{2}
+}
+func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseAddress.Merge(m, src)
+}
+func (m *DatabaseAddress) XXX_Size() int {
+	return m.Size()
+}
+func (m *DatabaseAddress) XXX_DiscardUnknown() {
+	xxx_messageInfo_DatabaseAddress.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DatabaseAddress proto.InternalMessageInfo
 
 func init() {
 	proto.RegisterType((*DatabaseRecord)(nil), "main.DatabaseRecord")
 	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
 	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
 }
+
+func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
+
+var fileDescriptor_b90fe3356ea5df07 = []byte{
+	// 270 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
+	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
+	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
+	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
+	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
+	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
+	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
+	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
+	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
+	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
+	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
+	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
+	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
+	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
+	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
+	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
+	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
+}
+
 func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -81,44 +180,51 @@ func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Addresses) > 0 {
+	if m.Missed != 0 {
-		for _, msg := range m.Addresses {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
-			dAtA[i] = 0xa
+		i--
-			i++
+		dAtA[i] = 0x20
-			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
-			n, err := msg.MarshalTo(dAtA[i:])
-			if err != nil {
-				return 0, err
-			}
-			i += n
-		}
-	}
-	if m.Misses != 0 {
-		dAtA[i] = 0x10
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
 	}
 	if m.Seen != 0 {
-		dAtA[i] = 0x18
-		i++
 		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
 	}
-	if m.Missed != 0 {
+	if m.Misses != 0 {
-		dAtA[i] = 0x20
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
-		i++
+		i--
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+		dAtA[i] = 0x10
 	}
-	return i, nil
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
 }
 
 func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -126,40 +232,48 @@ func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Key) > 0 {
+	if m.Seen != 0 {
-		dAtA[i] = 0xa
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
-		i++
+		i--
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		dAtA[i] = 0x18
-		i += copy(dAtA[i:], m.Key)
 	}
 	if len(m.Addresses) > 0 {
-		for _, msg := range m.Addresses {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
-			dAtA[i] = 0x12
+			{
-			i++
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
+				if err != nil {
-			n, err := msg.MarshalTo(dAtA[i:])
+					return 0, err
-			if err != nil {
+				}
-				return 0, err
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
 			}
-			i += n
+			i--
+			dAtA[i] = 0x12
 		}
 	}
-	if m.Seen != 0 {
+	if len(m.Key) > 0 {
-		dAtA[i] = 0x18
+		i -= len(m.Key)
-		i++
+		copy(dAtA[i:], m.Key)
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		i--
+		dAtA[i] = 0xa
 	}
-	return i, nil
+	return len(dAtA) - i, nil
 }
 
 func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -167,52 +281,45 @@ func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Address) > 0 {
-		dAtA[i] = 0xa
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
-		i += copy(dAtA[i:], m.Address)
-	}
 	if m.Expires != 0 {
-		dAtA[i] = 0x10
-		i++
 		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
+		i--
+		dAtA[i] = 0x10
 	}
-	return i, nil
+	if len(m.Address) > 0 {
+		i -= len(m.Address)
+		copy(dAtA[i:], m.Address)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
 }
 
-func encodeFixed64Database(dAtA []byte, offset int, v uint64) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	dAtA[offset+4] = uint8(v >> 32)
-	dAtA[offset+5] = uint8(v >> 40)
-	dAtA[offset+6] = uint8(v >> 48)
-	dAtA[offset+7] = uint8(v >> 56)
-	return offset + 8
-}
-func encodeFixed32Database(dAtA []byte, offset int, v uint32) int {
-	dAtA[offset] = uint8(v)
-	dAtA[offset+1] = uint8(v >> 8)
-	dAtA[offset+2] = uint8(v >> 16)
-	dAtA[offset+3] = uint8(v >> 24)
-	return offset + 4
-}
 func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
+	offset -= sovDatabase(v)
+	base := offset
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
 		v >>= 7
 		offset++
 	}
 	dAtA[offset] = uint8(v)
-	return offset + 1
+	return base
 }
 func (m *DatabaseRecord) Size() (n int) {
+	if m == nil {
+		return 0
+	}
 	var l int
 	_ = l
 	if len(m.Addresses) > 0 {
@@ -234,6 +341,9 @@ func (m *DatabaseRecord) Size() (n int) {
 }
 
 func (m *ReplicationRecord) Size() (n int) {
+	if m == nil {
+		return 0
+	}
 	var l int
 	_ = l
 	l = len(m.Key)
@@ -253,6 +363,9 @@ func (m *ReplicationRecord) Size() (n int) {
 }
 
 func (m *DatabaseAddress) Size() (n int) {
+	if m == nil {
+		return 0
+	}
 	var l int
 	_ = l
 	l = len(m.Address)
@@ -266,14 +379,7 @@ func (m *DatabaseAddress) Size() (n int) {
 }
 
 func sovDatabase(x uint64) (n int) {
-	for {
+	return (math_bits.Len64(x|1) + 6) / 7
-		n++
-		x >>= 7
-		if x == 0 {
-			break
-		}
-	}
-	return n
 }
 func sozDatabase(x uint64) (n int) {
 	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
@@ -293,7 +399,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -321,7 +427,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= (int(b) & 0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -330,6 +436,9 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -352,7 +461,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Misses |= (int32(b) & 0x7F) << shift
+				m.Misses |= int32(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -371,7 +480,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= (int64(b) & 0x7F) << shift
+				m.Seen |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -390,7 +499,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Missed |= (int64(b) & 0x7F) << shift
+				m.Missed |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -404,6 +513,9 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -431,7 +543,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -459,7 +571,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -469,6 +581,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -488,7 +603,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= (int(b) & 0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -497,6 +612,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -519,7 +637,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= (int64(b) & 0x7F) << shift
+				m.Seen |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -533,6 +651,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -560,7 +681,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -588,7 +709,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -598,6 +719,9 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -617,7 +741,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Expires |= (int64(b) & 0x7F) << shift
+				m.Expires |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -631,6 +755,9 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -646,6 +773,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 func skipDatabase(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0
+	depth := 0
 	for iNdEx < l {
 		var wire uint64
 		for shift := uint(0); ; shift += 7 {
@@ -677,10 +805,8 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			return iNdEx, nil
 		case 1:
 			iNdEx += 8
-			return iNdEx, nil
 		case 2:
 			var length int
 			for shift := uint(0); ; shift += 7 {
@@ -697,76 +823,34 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			iNdEx += length
 			if length < 0 {
 				return 0, ErrInvalidLengthDatabase
 			}
-			return iNdEx, nil
+			iNdEx += length
 		case 3:
-			for {
+			depth++
-				var innerWire uint64
-				var start int = iNdEx
-				for shift := uint(0); ; shift += 7 {
-					if shift >= 64 {
-						return 0, ErrIntOverflowDatabase
-					}
-					if iNdEx >= l {
-						return 0, io.ErrUnexpectedEOF
-					}
-					b := dAtA[iNdEx]
-					iNdEx++
-					innerWire |= (uint64(b) & 0x7F) << shift
-					if b < 0x80 {
-						break
-					}
-				}
-				innerWireType := int(innerWire & 0x7)
-				if innerWireType == 4 {
-					break
-				}
-				next, err := skipDatabase(dAtA[start:])
-				if err != nil {
-					return 0, err
-				}
-				iNdEx = start + next
-			}
-			return iNdEx, nil
 		case 4:
-			return iNdEx, nil
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupDatabase
+			}
+			depth--
 		case 5:
 			iNdEx += 4
-			return iNdEx, nil
 		default:
 			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
 		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthDatabase
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
 	}
-	panic("unreachable")
+	return 0, io.ErrUnexpectedEOF
 }
 
 var (
-	ErrInvalidLengthDatabase = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrInvalidLengthDatabase        = fmt.Errorf("proto: negative length found during unmarshaling")
-	ErrIntOverflowDatabase   = fmt.Errorf("proto: integer overflow")
+	ErrIntOverflowDatabase          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupDatabase = fmt.Errorf("proto: unexpected end of group")
 )
-
-func init() { proto.RegisterFile("database.proto", fileDescriptorDatabase) }
-
-var fileDescriptorDatabase = []byte{
-	// 264 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4b, 0x49, 0x2c, 0x49,
-	0x4c, 0x4a, 0x2c, 0x4e, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0xc9, 0x4d, 0xcc, 0xcc,
-	0x93, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf, 0x4f,
-	0xcf, 0xd7, 0x07, 0x4b, 0x26, 0x95, 0xa6, 0x81, 0x79, 0x60, 0x0e, 0x98, 0x05, 0xd1, 0xa4, 0xd4,
-	0xcf, 0xc8, 0xc5, 0xe7, 0x02, 0x35, 0x27, 0x28, 0x35, 0x39, 0xbf, 0x28, 0x45, 0xc8, 0x92, 0x8b,
-	0x33, 0x31, 0x25, 0xa5, 0x28, 0xb5, 0xb8, 0x38, 0xb5, 0x58, 0x82, 0x51, 0x81, 0x59, 0x83, 0xdb,
-	0x48, 0x54, 0x0f, 0x64, 0xb6, 0x1e, 0x4c, 0xa1, 0x23, 0x44, 0xda, 0x89, 0xe5, 0xc4, 0x3d, 0x79,
-	0x86, 0x20, 0x84, 0x6a, 0x21, 0x31, 0x2e, 0xb6, 0xdc, 0x4c, 0xb0, 0x3e, 0x26, 0x05, 0x46, 0x0d,
-	0xd6, 0x20, 0x28, 0x4f, 0x48, 0x88, 0x8b, 0xa5, 0x38, 0x35, 0x35, 0x4f, 0x82, 0x59, 0x81, 0x51,
-	0x83, 0x39, 0x08, 0xcc, 0x86, 0xab, 0x4d, 0x91, 0x60, 0x01, 0x8b, 0x42, 0x79, 0x4a, 0x25, 0x5c,
-	0x82, 0x41, 0xa9, 0x05, 0x39, 0x99, 0xc9, 0x89, 0x25, 0x99, 0xf9, 0x79, 0x50, 0x37, 0x09, 0x70,
-	0x31, 0x67, 0xa7, 0x56, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0x70, 0x06, 0x81, 0x98, 0xa8, 0xae, 0x64,
-	0x22, 0xc9, 0x95, 0x58, 0x5c, 0xa3, 0xe4, 0xca, 0xc5, 0x8f, 0xa6, 0x4f, 0x48, 0x82, 0x8b, 0x1d,
-	0xaa, 0x07, 0x6a, 0x2f, 0x8c, 0x0b, 0x92, 0x49, 0xad, 0x28, 0xc8, 0x2c, 0x82, 0xfa, 0x93, 0x39,
-	0x08, 0xc6, 0x75, 0x12, 0x38, 0xf1, 0x50, 0x8e, 0xe1, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x8f, 0xe4,
-	0x18, 0x1f, 0x3c, 0x92, 0x63, 0x4c, 0x62, 0x03, 0x87, 0xb3, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff,
-	0x6a, 0x22, 0xa2, 0x85, 0xae, 0x01, 0x00, 0x00,
-}

cmd/stdiscosrv/database.proto

@@ -8,9 +8,12 @@ syntax = "proto3";
 
 package main;
 
-import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+import "repos/protobuf/gogoproto/gogo.proto";
 
 option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.goproto_unkeyed_all) = false;
+option (gogoproto.goproto_unrecognized_all) = false;
+option (gogoproto.goproto_sizecache_all) = false;
 
 message DatabaseRecord {
     repeated DatabaseAddress addresses = 1 [(gogoproto.nullable) = false];

cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv

@@ -0,0 +1,4 @@
+[stdiscosrv]
+title=Syncthing discovery server
+description=Lets syncthing clients discover each other
+ports=8443/tcp

cmd/stdiscosrv/etc/linux-systemd/default

@@ -0,0 +1,3 @@
+# Default settings for syncthing-discosrv (stdiscosrv).
+## Add Options here:
+DISCOSRV_OPTS=

cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=Syncthing Discovery Server
+After=network.target
+Documentation=man:stdiscosrv(1)
+
+[Service]
+WorkingDirectory=/var/lib/syncthing-discosrv
+EnvironmentFile=/etc/default/syncthing-discosrv
+ExecStart=/usr/bin/stdiscosrv $DISCOSRV_OPTS
+
+# Hardening
+User=syncthing-discosrv
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-discosrv
+NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
+
+[Install]
+WantedBy=multi-user.target
+Alias=syncthing-discosrv.service

cmd/stdiscosrv/main.go

@@ -9,17 +9,15 @@ package main
 import (
 	"crypto/tls"
 	"flag"
-	"fmt"
 	"log"
 	"net"
 	"net/http"
 	"os"
-	"runtime"
-	"strconv"
 	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -65,34 +63,11 @@ var levelDBOptions = &opt.Options{
 	WriteBuffer: 32 << 20, // default 4<<20
 }
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`stdiscosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
-}
-
 var (
 	debug = false
 )
 
 func main() {
-	const (
-		cleanIntv = 1 * time.Hour
-		statsIntv = 5 * time.Minute
-	)
-
 	var listen string
 	var dir string
 	var metricsListen string
@@ -114,19 +89,24 @@ func main() {
 	flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
 	flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
 	flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
-	log.Println(LongVersion)
+	log.Println(build.LongVersionFor("stdiscosrv"))
+	if *showVersion {
+		return
+	}
 
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
-	if err != nil {
+	if os.IsNotExist(err) {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 0)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
+	} else if err != nil {
+		log.Fatalln("Failed to load keypair:", err)
 	}
-
 	devID := protocol.NewDeviceID(cert.Certificate[0])
 	log.Println("Server device ID is", devID)
 

cmd/stdiscosrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-discosrv --ingroup syncthing syncthing-discosrv

cmd/stdiscosrv/stats.go

@@ -109,5 +109,15 @@ func init() {
 		databaseKeys, databaseStatisticsSeconds,
 		databaseOperations, databaseOperationSeconds)
 
-	prometheus.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "syncthing_discovery"))
+	processCollectorOpts := prometheus.ProcessCollectorOpts{
+		Namespace: "syncthing_discovery",
+		PidFn: func() (int, error) {
+			return os.Getpid(), nil
+		},
+	}
+
+	prometheus.MustRegister(
+		prometheus.NewProcessCollector(processCollectorOpts),
+	)
+
 }

cmd/stfinddevice/main.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"errors"
 	"flag"
@@ -17,6 +18,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -82,7 +84,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 }
 
 func checkServer(deviceID protocol.DeviceID, server string) checkResult {
-	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger)
 	if err != nil {
 		return checkResult{error: err}
 	}
@@ -94,7 +96,7 @@ func checkServer(deviceID protocol.DeviceID, server string) checkResult {
 	})
 
 	go func() {
-		addresses, err := disco.Lookup(deviceID)
+		addresses, err := disco.Lookup(context.Background(), deviceID)
 		res <- checkResult{addresses: addresses, error: err}
 	}()
 

cmd/stgenfiles/main.go

@@ -82,7 +82,7 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
 		return err
 	}
 
-	_ = os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
+	os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
 
 	t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
 	return os.Chtimes(p1, t, t)

cmd/stindex/accounting.go

@@ -0,0 +1,58 @@
+// Copyright (C) 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"text/tabwriter"
+
+	"github.com/syncthing/syncthing/lib/db/backend"
+)
+
+// account prints key and data size statistics per class
+func account(ldb backend.Backend) {
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var ksizes [256]int
+	var dsizes [256]int
+	var counts [256]int
+	var max [256]int
+
+	for it.Next() {
+		key := it.Key()
+		t := key[0]
+		ds := len(it.Value())
+		ks := len(key)
+		s := ks + ds
+
+		counts[t]++
+		ksizes[t] += ks
+		dsizes[t] += ds
+		if s > max[t] {
+			max[t] = s
+		}
+	}
+
+	tw := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', tabwriter.AlignRight)
+	toti, totds, totks := 0, 0, 0
+	for t := range ksizes {
+		if ksizes[t] > 0 {
+			// yes metric kilobytes 🤘
+			fmt.Fprintf(tw, "0x%02x:\t%d items,\t%d KB keys +\t%d KB data,\t%d B +\t%d B avg,\t%d B max\t\n", t, counts[t], ksizes[t]/1000, dsizes[t]/1000, ksizes[t]/counts[t], dsizes[t]/counts[t], max[t])
+			toti += counts[t]
+			totds += dsizes[t]
+			totks += ksizes[t]
+		}
+	}
+	fmt.Fprintf(tw, "Total\t%d items,\t%d KB keys +\t%d KB data.\t\n", toti, totks/1000, totds/1000)
+	tw.Flush()
+}

cmd/stindex/dump.go

@@ -13,11 +13,15 @@ import (
 	"time"
 
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func dump(ldb *db.Instance) {
+func dump(ldb backend.Backend) {
-	it := ldb.NewIterator(nil, nil)
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {
@@ -76,6 +80,31 @@ func dump(ldb *db.Instance) {
 				fmt.Printf("[deviceidx] K:%d V:%s\n", key, dev)
 			}
 
+		case db.KeyTypeIndexID:
+			device := binary.BigEndian.Uint32(it.Key()[1:])
+			folder := binary.BigEndian.Uint32(it.Key()[5:])
+			fmt.Printf("[indexid] D:%d F:%d I:%x\n", device, folder, it.Value())
+
+		case db.KeyTypeFolderMeta:
+			folder := binary.BigEndian.Uint32(it.Key()[1:])
+			fmt.Printf("[foldermeta] F:%d V:%x\n", folder, it.Value())
+
+		case db.KeyTypeMiscData:
+			fmt.Printf("[miscdata] K:%q V:%q\n", it.Key()[1:], it.Value())
+
+		case db.KeyTypeSequence:
+			folder := binary.BigEndian.Uint32(it.Key()[1:])
+			seq := binary.BigEndian.Uint64(it.Key()[5:])
+			fmt.Printf("[sequence] F:%d S:%d V:%q\n", folder, seq, it.Value())
+
+		case db.KeyTypeNeed:
+			folder := binary.BigEndian.Uint32(it.Key()[1:])
+			file := string(it.Key()[5:])
+			fmt.Printf("[need] F:%d V:%q\n", folder, file)
+
+		case db.KeyTypeBlockList:
+			fmt.Printf("[blocklist] H:%x\n", it.Key()[1:])
+
 		default:
 			fmt.Printf("[???]\n  %x\n  %x\n", it.Key(), it.Value())
 		}

cmd/stindex/dumpsize.go

@@ -10,8 +10,10 @@ import (
 	"container/heap"
 	"encoding/binary"
 	"fmt"
+	"log"
 
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
 )
 
 type SizedElement struct {
@@ -37,11 +39,14 @@ func (h *ElementHeap) Pop() interface{} {
 	return x
 }
 
-func dumpsize(ldb *db.Instance) {
+func dumpsize(ldb backend.Backend) {
 	h := &ElementHeap{}
 	heap.Init(h)
 
-	it := ldb.NewIterator(nil, nil)
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
 	var ele SizedElement
 	for it.Next() {
 		key := it.Key()

cmd/stindex/idxck.go

@@ -0,0 +1,296 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"log"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+type fileInfoKey struct {
+	folder uint32
+	device uint32
+	name   string
+}
+
+type globalKey struct {
+	folder uint32
+	name   string
+}
+
+type sequenceKey struct {
+	folder   uint32
+	sequence uint64
+}
+
+func idxck(ldb backend.Backend) (success bool) {
+	folders := make(map[uint32]string)
+	devices := make(map[uint32]string)
+	deviceToIDs := make(map[string]uint32)
+	fileInfos := make(map[fileInfoKey]protocol.FileInfo)
+	globals := make(map[globalKey]db.VersionList)
+	sequences := make(map[sequenceKey]string)
+	needs := make(map[globalKey]struct{})
+	blocklists := make(map[string]struct{})
+	usedBlocklists := make(map[string]struct{})
+	var localDeviceKey uint32
+	success = true
+
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+	for it.Next() {
+		key := it.Key()
+		switch key[0] {
+		case db.KeyTypeDevice:
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+
+			var f protocol.FileInfo
+			err := f.Unmarshal(it.Value())
+			if err != nil {
+				fmt.Println("Unable to unmarshal FileInfo:", err)
+				success = false
+				continue
+			}
+
+			fileInfos[fileInfoKey{folder, device, name}] = f
+
+		case db.KeyTypeGlobal:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			var flv db.VersionList
+			if err := flv.Unmarshal(it.Value()); err != nil {
+				fmt.Println("Unable to unmarshal VersionList:", err)
+				success = false
+				continue
+			}
+			globals[globalKey{folder, name}] = flv
+
+		case db.KeyTypeFolderIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			folders[key] = string(it.Value())
+
+		case db.KeyTypeDeviceIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			devices[key] = string(it.Value())
+			deviceToIDs[string(it.Value())] = key
+			if bytes.Equal(it.Value(), protocol.LocalDeviceID[:]) {
+				localDeviceKey = key
+			}
+
+		case db.KeyTypeSequence:
+			folder := binary.BigEndian.Uint32(key[1:])
+			seq := binary.BigEndian.Uint64(key[5:])
+			val := it.Value()
+			sequences[sequenceKey{folder, seq}] = string(val[9:])
+
+		case db.KeyTypeNeed:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			needs[globalKey{folder, name}] = struct{}{}
+
+		case db.KeyTypeBlockList:
+			hash := string(key[1:])
+			blocklists[hash] = struct{}{}
+		}
+	}
+
+	if localDeviceKey == 0 {
+		fmt.Println("Missing key for local device in device index (bailing out)")
+		success = false
+		return
+	}
+
+	var missingSeq []sequenceKey
+	for fk, fi := range fileInfos {
+		if fk.name != fi.Name {
+			fmt.Printf("Mismatching FileInfo name, %q (key) != %q (actual)\n", fk.name, fi.Name)
+			success = false
+		}
+
+		folder := folders[fk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for FileInfo %q\n", fk.folder, fk.name)
+			success = false
+			continue
+		}
+		if devices[fk.device] == "" {
+			fmt.Printf("Unknown device ID %d for FileInfo %q, folder %q\n", fk.folder, fk.name, folder)
+			success = false
+		}
+
+		if fk.device == localDeviceKey {
+			sk := sequenceKey{fk.folder, uint64(fi.Sequence)}
+			name, ok := sequences[sk]
+			if !ok {
+				fmt.Printf("Sequence entry missing for FileInfo %q, folder %q, seq %d\n", fi.Name, folder, fi.Sequence)
+				missingSeq = append(missingSeq, sk)
+				success = false
+				continue
+			}
+			if name != fi.Name {
+				fmt.Printf("Sequence entry refers to wrong name, %q (seq) != %q (FileInfo), folder %q, seq %d\n", name, fi.Name, folder, fi.Sequence)
+				success = false
+			}
+		}
+
+		if len(fi.Blocks) == 0 && len(fi.BlocksHash) != 0 {
+			key := string(fi.BlocksHash)
+			if _, ok := blocklists[key]; !ok {
+				fmt.Printf("Missing block list for file %q, block list hash %x\n", fi.Name, fi.BlocksHash)
+				success = false
+			} else {
+				usedBlocklists[key] = struct{}{}
+			}
+		}
+	}
+
+	// Aggregate the ranges of missing sequence entries, print them
+
+	sort.Slice(missingSeq, func(a, b int) bool {
+		if missingSeq[a].folder != missingSeq[b].folder {
+			return missingSeq[a].folder < missingSeq[b].folder
+		}
+		return missingSeq[a].sequence < missingSeq[b].sequence
+	})
+
+	var folder uint32
+	var startSeq, prevSeq uint64
+	for _, sk := range missingSeq {
+		if folder != sk.folder || sk.sequence != prevSeq+1 {
+			if folder != 0 {
+				fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
+			}
+			startSeq = sk.sequence
+			folder = sk.folder
+		}
+		prevSeq = sk.sequence
+	}
+	if folder != 0 {
+		fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
+	}
+
+	for gk, vl := range globals {
+		folder := folders[gk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
+			success = false
+		}
+		for i, fv := range vl.Versions {
+			dev, ok := deviceToIDs[string(fv.Device)]
+			if !ok {
+				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, fv.Device)
+				success = false
+			}
+			fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
+			if !ok {
+				fmt.Printf("VersionList %q, folder %q, entry %d refers to unknown FileInfo\n", gk.name, folder, i)
+				success = false
+			}
+			if !fi.Version.Equal(fv.Version) {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Version, fi.Version)
+				success = false
+			}
+			if fi.IsInvalid() != fv.Invalid {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Invalid, fi.IsInvalid())
+				success = false
+			}
+		}
+
+		// If we need this file we should have a need entry for it. False
+		// positives from needsLocally for deleted files, where we might
+		// legitimately lack an entry if we never had it, and ignored files.
+		if needsLocally(vl) {
+			_, ok := needs[gk]
+			if !ok {
+				dev := deviceToIDs[string(vl.Versions[0].Device)]
+				fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
+				if !fi.IsDeleted() && !fi.IsIgnored() {
+					fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
+				}
+			}
+		}
+	}
+
+	seenSeq := make(map[fileInfoKey]uint64)
+	for sk, name := range sequences {
+		folder := folders[sk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for sequence entry %d, %q\n", sk.folder, sk.sequence, name)
+			success = false
+			continue
+		}
+
+		if prev, ok := seenSeq[fileInfoKey{folder: sk.folder, name: name}]; ok {
+			fmt.Printf("Duplicate sequence entry for %q, folder %q, seq %d (prev %d)\n", name, folder, sk.sequence, prev)
+			success = false
+		}
+		seenSeq[fileInfoKey{folder: sk.folder, name: name}] = sk.sequence
+
+		fi, ok := fileInfos[fileInfoKey{sk.folder, localDeviceKey, name}]
+		if !ok {
+			fmt.Printf("Missing FileInfo for sequence entry %d, folder %q, %q\n", sk.sequence, folder, name)
+			success = false
+			continue
+		}
+		if fi.Sequence != int64(sk.sequence) {
+			fmt.Printf("Sequence mismatch for %q, folder %q, %d (key) != %d (FileInfo)\n", name, folder, sk.sequence, fi.Sequence)
+			success = false
+		}
+	}
+
+	for nk := range needs {
+		folder := folders[nk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for need entry %q\n", nk.folder, nk.name)
+			success = false
+			continue
+		}
+
+		vl, ok := globals[nk]
+		if !ok {
+			fmt.Printf("Missing global for need entry %q, folder %q\n", nk.name, folder)
+			success = false
+			continue
+		}
+
+		if !needsLocally(vl) {
+			fmt.Printf("Need entry for file we don't need, %q, folder %q\n", nk.name, folder)
+			success = false
+		}
+	}
+
+	if d := len(blocklists) - len(usedBlocklists); d > 0 {
+		fmt.Printf("%d block list entries out of %d needs GC\n", d, len(blocklists))
+	}
+
+	return
+}
+
+func needsLocally(vl db.VersionList) bool {
+	var lv *protocol.Vector
+	for _, fv := range vl.Versions {
+		if bytes.Equal(fv.Device, protocol.LocalDeviceID[:]) {
+			lv = &fv.Version
+			break
+		}
+	}
+	if lv == nil {
+		return true // proviosinally, it looks like we need the file
+	}
+	return !lv.GreaterEqual(vl.Versions[0].Version)
+}

cmd/stindex/main.go

@@ -13,7 +13,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
 )
 
 func main() {
@@ -21,7 +21,7 @@ func main() {
 	log.SetFlags(0)
 	log.SetOutput(os.Stdout)
 
-	flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize")
+	flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize, idxck")
 
 	flag.Parse()
 
@@ -30,18 +30,23 @@ func main() {
 		path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
 	}
 
-	fmt.Println("Path:", path)
+	ldb, err := backend.OpenLevelDBRO(path)
-
-	ldb, err := db.Open(path)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	if mode == "dump" {
+	switch mode {
+	case "dump":
 		dump(ldb)
-	} else if mode == "dumpsize" {
+	case "dumpsize":
 		dumpsize(ldb)
-	} else {
+	case "idxck":
+		if !idxck(ldb) {
+			os.Exit(1)
+		}
+	case "account":
+		account(ldb)
+	default:
 		fmt.Println("Unknown mode")
 	}
 }

cmd/strelaypoolsrv/gui/index.html

@@ -9,8 +9,15 @@
     <meta name="author" content=""/>
 
     <title>Relay stats</title>
-    <link href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
+    <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"/>
-    <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.0.13/css/all.css"/>
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
+    <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"/>
+    <link rel="stylesheet" href="https://unpkg.com/leaflet@1.6.0/dist/leaflet.css"
+  integrity="sha512-xwE/Az9zrjBIphAcBb3F6JVqxf46+CDLwfLMHloNu6KEQCAWi6HcDUbeOfBIptF7tcCzusKFjFw2yuvEpDL9wQ=="
+  crossorigin=""/>
+  <script src="https://unpkg.com/leaflet@1.6.0/dist/leaflet.js"
+  integrity="sha512-gZwIG9x3wUXg2hdXF6+rVkLF/0Vi9U8D2Ntg4Ga5I5BZpVkVxlJWbSQtXPSiUTtC0TjtGOmxa1AJPuV0CPthew=="
+  crossorigin=""></script>
 
     <style>
       #map {
@@ -38,7 +45,7 @@
     <div class="container">
       <h1>Relay Pool Data</h1>
       <div ng-if="relays === undefined" class="text-center">
-        <img src="//cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
+        <img src="https://cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
         <p>Please wait while we gather data</p>
       </div>
       <div>
@@ -184,18 +191,17 @@
     </div>
 
 
-    <script type="text/javascript" src="//code.jquery.com/jquery-2.1.4.min.js"></script>
+    <script type="text/javascript" src="https://code.jquery.com/jquery-2.1.4.min.js"></script>
-    <script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
+    <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
-    <script type="text/javascript" src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
+    <script type="text/javascript" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
-    <script type="text/javascript" src="//maps.googleapis.com/maps/api/js"></script>
   </body>
 
   <script>
   angular.module('syncthing', [
   ])
-  .config(function($httpProvider) {
+  .config(['$httpProvider', function($httpProvider) {
     $httpProvider.defaults.timeout = 5000;
-  })
+  }])
   .filter('bytes', function() {
     return function(bytes, precision) {
       if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) return '-';
@@ -228,11 +234,12 @@
       numProxies: 0,
       uptimeSeconds: 0,
     };
-    $scope.map = new google.maps.Map(document.getElementById('map'), {
+    $scope.map = L.map('map').setView([40.90296, 1.90925], 2);
-      zoom: 1,
+    L.tileLayer('http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',
-      mapTypeId: google.maps.MapTypeId.ROADMAP
+    {
-    });
+      attribution: 'Leaflet',
-    $scope.mapBounds = new google.maps.LatLngBounds();
+      maxZoom: 17
+    }).addTo($scope.map);
     $scope.tooltipTemplate = $('#infoTemplate').html();
     $scope.usedLocations = {};
     $scope.sortType = 'stats.numActiveSessions';
@@ -279,8 +286,9 @@
         }
       });
 
-      $scope.map.fitBounds($scope.mapBounds);
       if ($scope.relays.length == 1) {
+        //Center to only relay with zoom
+        $scope.map.panTo(new L.LatLng(relays[0].location.latitude, relays[0].location.longitude));
         $scope.map.setZoom(13);
       }
     });
@@ -300,44 +308,50 @@
 
         var locParts = loc.split(',');
 
-        relay.marker = new google.maps.Marker({
+        relay.marker = new L.Marker([relay.location.latitude, relay.location.longitude],{
-          map: $scope.map,
-          position: new google.maps.LatLng(locParts[0], locParts[1]),
           title: relay.url,
         });
 
         var scope = $rootScope.$new(true);
         scope.relay = relay;
 
-        relay.marker.info = new google.maps.InfoWindow({
+        var icon = new L.Icon({
-          content: $compile($scope.tooltipTemplate)(scope)[0],
+          iconSize:     [18, 28], // size of the icon
+          iconAnchor:   [9, 28], // point of the icon which will correspond to marker's location
+          shadowAnchor: [0, 0],  // the same for the shadow
+          popupAnchor:  [0, -27], // popup anchor
+          shadowSize:   [0,0],
+          iconUrl: 'https://cdn.rawgit.com/pointhi/leaflet-color-markers/master/img/marker-icon-red.png',
+          shadowUrl: 'https://cdnjs.cloudflare.com/ajax/libs/leaflet/0.7.7/images/marker-shadow.png',
         });
 
+        relay.marker = new L.marker(new L.latLng(locParts[0], locParts[1]),{icon})
+                            .bindPopup($compile($scope.tooltipTemplate)(scope)[0],{})
+                            .on('mouseover', function (e) {
+                                this.openPopup();
+                            }).on('mouseout', function (e) {
+                                this.closePopup();
+                            }).addTo($scope.map);
+
         relay.showMarker = function() {
-          relay.marker.info.open($scope.map, relay.marker);
+          relay.marker.openPopup();
         }
-
+        
         relay.hideMarker = function() {
-          relay.marker.info.close();
+          relay.marker.closePopup();
         }
+      }
 
-        relay.marker.addListener('mouseover', relay.showMarker);
-        relay.marker.addListener('mouseout', relay.hideMarker);
-
-        $scope.mapBounds.extend(relay.marker.position);
-    }
 
     function addCircleToMap(relay) {
-      relay.marker.circle = new google.maps.Circle({
+      console.log(relay.location.latitude)
-        strokeColor: '#FF0000',
+      L.circle([relay.location.latitude, relay.location.longitude], 
-        strokeOpacity: 0.8,
+        {
-        strokeWeight: 2,
+          radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000,
-        fillColor: '#FF0000',
+          color: "FF0000",
-        fillOpacity: 0.35,
+          fillColor: "#FF0000",
-        map: $scope.map,
+          fillOpacity: 0.35,
-        center: relay.marker.position,
+        }).addTo($scope.map);
-        radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000
-      });
     }
 
     function constructURI(url) {

cmd/strelaypoolsrv/main.go

@@ -1,20 +1,17 @@
 // Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
 
-//go:generate go run ../../script/genassets.go gui >auto/gui.go
-
 package main
 
 import (
-	"bytes"
 	"compress/gzip"
+	"context"
 	"crypto/tls"
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
-	"math/rand"
-	"mime"
 	"net"
 	"net/http"
 	"net/url"
@@ -29,6 +26,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
+	"github.com/syncthing/syncthing/lib/assets"
+	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/relay/client"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -92,33 +91,35 @@ type result struct {
 }
 
 var (
-	testCert        tls.Certificate
+	testCert          tls.Certificate
-	knownRelaysFile = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
+	knownRelaysFile   = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
-	listen          = ":80"
+	listen            = ":80"
-	dir             string
+	dir               string
-	evictionTime    = time.Hour
+	evictionTime      = time.Hour
-	debug           bool
+	debug             bool
-	getLRUSize      = 10 << 10
+	getLRUSize        = 10 << 10
-	getLimitBurst   = 10
+	getLimitBurst     = 10
-	getLimitAvg     = 2
+	getLimitAvg       = 2
-	postLRUSize     = 1 << 10
+	postLRUSize       = 1 << 10
-	postLimitBurst  = 2
+	postLimitBurst    = 2
-	postLimitAvg    = 2
+	postLimitAvg      = 2
-	getLimit        time.Duration
+	getLimit          time.Duration
-	postLimit       time.Duration
+	postLimit         time.Duration
-	permRelaysFile  string
+	permRelaysFile    string
-	ipHeader        string
+	ipHeader          string
-	geoipPath       string
+	geoipPath         string
-	proto           string
+	proto             string
-	statsRefresh    = time.Minute / 2
+	statsRefresh      = time.Minute / 2
+	requestQueueLen   = 10
+	requestProcessors = 1
 
-	getMut      = sync.NewRWMutex()
+	getMut      = sync.NewMutex()
 	getLRUCache *lru.Cache
 
-	postMut      = sync.NewRWMutex()
+	postMut      = sync.NewMutex()
 	postLRUCache *lru.Cache
 
-	requests = make(chan request, 10)
+	requests chan request
 
 	mut             = sync.NewRWMutex()
 	knownRelays     = make([]*relay, 0)
@@ -131,6 +132,9 @@ const (
 )
 
 func main() {
+	log.SetOutput(os.Stdout)
+	log.SetFlags(log.Lshortfile)
+
 	flag.StringVar(&listen, "listen", listen, "Listen address")
 	flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
 	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
@@ -146,9 +150,13 @@ func main() {
 	flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
 	flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
 	flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
+	flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
+	flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
 
 	flag.Parse()
 
+	requests = make(chan request, requestQueueLen)
+
 	getLimit = 10 * time.Second / time.Duration(getLimitAvg)
 	postLimit = time.Minute / time.Duration(postLimitAvg)
 
@@ -164,7 +172,9 @@ func main() {
 
 	testCert = createTestCertificate()
 
-	go requestProcessor()
+	for i := 0; i < requestProcessors; i++ {
+		go requestProcessor()
+	}
 
 	// Load relays from cache in the background.
 	// Load them in a serial fashion to make sure any genuine requests
@@ -253,88 +263,31 @@ func handleMetrics(w http.ResponseWriter, r *http.Request) {
 func handleAssets(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Cache-Control", "no-cache, must-revalidate")
 
-	assets := auto.Assets()
 	path := r.URL.Path[1:]
 	if path == "" {
 		path = "index.html"
 	}
 
-	bs, ok := assets[path]
+	content, ok := auto.Assets()[path]
 	if !ok {
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
 
-	etag := fmt.Sprintf("%d", auto.Generated)
+	assets.Serve(w, r, assets.Asset{
-	modified := time.Unix(auto.Generated, 0).UTC()
+		ContentGz: content,
-
+		Filename:  path,
-	w.Header().Set("Last-Modified", modified.Format(http.TimeFormat))
+		Modified:  time.Unix(auto.Generated, 0).UTC(),
-	w.Header().Set("Etag", etag)
+	})
-
-	mtype := mimeTypeForFile(path)
-	if len(mtype) != 0 {
-		w.Header().Set("Content-Type", mtype)
-	}
-
-	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modified.Add(time.Second).After(t) {
-		w.WriteHeader(http.StatusNotModified)
-		return
-	}
-
-	if match := r.Header.Get("If-None-Match"); match != "" {
-		if strings.Contains(match, etag) {
-			w.WriteHeader(http.StatusNotModified)
-			return
-		}
-	}
-	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
-		w.Header().Set("Content-Encoding", "gzip")
-	} else {
-		// ungzip if browser not send gzip accepted header
-		var gr *gzip.Reader
-		gr, _ = gzip.NewReader(bytes.NewReader(bs))
-		bs, _ = ioutil.ReadAll(gr)
-		gr.Close()
-	}
-	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs)))
-
-	w.Write(bs)
-}
-
-func mimeTypeForFile(file string) string {
-	// We use a built in table of the common types since the system
-	// TypeByExtension might be unreliable. But if we don't know, we delegate
-	// to the system.
-	ext := filepath.Ext(file)
-	switch ext {
-	case ".htm", ".html":
-		return "text/html"
-	case ".css":
-		return "text/css"
-	case ".js":
-		return "application/javascript"
-	case ".json":
-		return "application/json"
-	case ".png":
-		return "image/png"
-	case ".ttf":
-		return "application/x-font-ttf"
-	case ".woff":
-		return "application/x-font-woff"
-	case ".svg":
-		return "image/svg+xml"
-	default:
-		return mime.TypeByExtension(ext)
-	}
 }
 
 func handleRequest(w http.ResponseWriter, r *http.Request) {
 	timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
 
-	lw := NewLoggingResponseWriter(w)
+	w = NewLoggingResponseWriter(w)
-
 	defer func() {
 		timer.ObserveDuration()
+		lw := w.(*loggingResponseWriter)
 		apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
 	}()
 
@@ -363,19 +316,24 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func handleGetRequest(w http.ResponseWriter, r *http.Request) {
+func handleGetRequest(rw http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
 	mut.RLock()
 	relays := append(permanentRelays, knownRelays...)
 	mut.RUnlock()
 
 	// Shuffle
-	for i := range relays {
+	rand.Shuffle(relays)
-		j := rand.Intn(i + 1)
+
-		relays[i], relays[j] = relays[j], relays[i]
+	w := io.Writer(rw)
+	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+		rw.Header().Set("Content-Encoding", "gzip")
+		gw := gzip.NewWriter(rw)
+		defer gw.Close()
+		w = gw
 	}
 
-	json.NewEncoder(w).Encode(map[string][]*relay{
+	_ = json.NewEncoder(w).Encode(map[string][]*relay{
 		"relays": relays,
 	})
 }
@@ -389,7 +347,7 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		if debug {
 			log.Println("Failed to parse payload")
 		}
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
@@ -398,7 +356,7 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		if debug {
 			log.Println("Failed to parse URI", newRelay.URL)
 		}
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
@@ -407,7 +365,7 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		if debug {
 			log.Println("Failed to split URI", newRelay.URL)
 		}
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
@@ -483,11 +441,11 @@ func handleRelayTest(request request) {
 	if debug {
 		log.Println("Request for", request.relay)
 	}
-	if !client.TestRelay(request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
+	if err := client.TestRelay(context.TODO(), request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3); err != nil {
 		if debug {
-			log.Println("Test for relay", request.relay, "failed")
+			log.Println("Test for relay", request.relay, "failed:", err)
 		}
-		request.result <- result{fmt.Errorf("connection test failed"), 0}
+		request.result <- result{err, 0}
 		return
 	}
 
@@ -565,26 +523,21 @@ func evict(relay *relay) func() {
 	}
 }
 
-func limit(addr string, cache *lru.Cache, lock sync.RWMutex, intv time.Duration, burst int) bool {
+func limit(addr string, cache *lru.Cache, lock sync.Mutex, intv time.Duration, burst int) bool {
 	if host, _, err := net.SplitHostPort(addr); err == nil {
 		addr = host
 	}
 
-	lock.RLock()
+	lock.Lock()
-	bkt, ok := cache.Get(addr)
+	v, _ := cache.Get(addr)
-	lock.RUnlock()
+	bkt, ok := v.(*rate.Limiter)
-	if ok {
+	if !ok {
-		bkt := bkt.(*rate.Limiter)
+		bkt = rate.NewLimiter(rate.Every(intv), burst)
-		if !bkt.Allow() {
+		cache.Add(addr, bkt)
-			// Rate limit
-			return true
-		}
-	} else {
-		lock.Lock()
-		cache.Add(addr, rate.NewLimiter(rate.Every(intv), burst))
-		lock.Unlock()
 	}
-	return false
+	lock.Unlock()
+
+	return !bkt.Allow()
 }
 
 func loadRelays(file string) []*relay {
@@ -636,7 +589,7 @@ func createTestCertificate() tls.Certificate {
 	}
 
 	certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
-	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 3072)
+	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 20*365)
 	if err != nil {
 		log.Fatalln("Failed to create test X509 key pair:", err)
 	}

cmd/strelaypoolsrv/stats.go

@@ -14,7 +14,16 @@ import (
 )
 
 func init() {
-	prometheus.MustRegister(prometheus.NewProcessCollector(os.Getpid(), "syncthing_relaypoolsrv"))
+	processCollectorOpts := prometheus.ProcessCollectorOpts{
+		Namespace: "syncthing_relaypoolsrv",
+		PidFn: func() (int, error) {
+			return os.Getpid(), nil
+		},
+	}
+
+	prometheus.MustRegister(
+		prometheus.NewProcessCollector(processCollectorOpts),
+	)
 }
 
 var (
@@ -117,7 +126,7 @@ func refreshStats() {
 		go func(rel *relay) {
 			t0 := time.Now()
 			stats := fetchStats(rel)
-			duration := time.Now().Sub(t0).Seconds()
+			duration := time.Since(t0).Seconds()
 			result := "success"
 			if stats == nil {
 				result = "failed"

cmd/strelaysrv/etc/firewall-ufw/strelaysrv

@@ -0,0 +1,9 @@
+[strelaysrv]
+title=Syncthing relay server
+description=Proxies traffic of syncthing client behind firewalls
+ports=22067/tcp
+
+[strelaysrv-metrics]
+title=Syncthing relay metrics
+description=Provides metrics about the syncthing relay server
+ports=22070/tcp

cmd/strelaysrv/etc/linux-systemd/default

@@ -0,0 +1,5 @@
+# Default settings for syncthing-relaysrv (strelaysrv).
+NAT=true
+
+## Add Options here:
+RELAYSRV_OPTS=

cmd/strelaysrv/etc/linux-systemd/strelaysrv.service

@@ -1,17 +1,25 @@
 [Unit]
-Description=Syncthing relay server
+Description=Syncthing Relay Server
 After=network.target
+Documentation=man:strelaysrv(1)
 
 [Service]
-User=strelaysrv
+WorkingDirectory=/var/lib/syncthing-relaysrv
-Group=strelaysrv
+EnvironmentFile=/etc/default/syncthing-relaysrv
-ExecStart=/usr/bin/strelaysrv
+ExecStart=/usr/bin/strelaysrv -nat=${NAT} $RELAYSRV_OPTS
-WorkingDirectory=/var/lib/strelaysrv
 
-PrivateTmp=true
+# Hardening
-ProtectSystem=full
+User=syncthing-relaysrv
-ProtectHome=true
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-relaysrv
 NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
 
 [Install]
 WantedBy=multi-user.target
+Alias=syncthing-relaysrv.service

cmd/strelaysrv/main.go

@@ -14,12 +14,13 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
 	"sync/atomic"
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/relay/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -33,24 +34,6 @@ import (
 	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
 )
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`strelaysrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
-}
-
 var (
 	listen string
 	debug  bool
@@ -116,8 +99,15 @@ func main() {
 	flag.IntVar(&natTimeout, "nat-timeout", 10, "NAT discovery timeout in seconds")
 	flag.BoolVar(&pprofEnabled, "pprof", false, "Enable the built in profiling on the status server")
 	flag.IntVar(&networkBufferSize, "network-buffer", 2048, "Network buffer size (two of these per proxied connection)")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
+	longVer := build.LongVersionFor("strelaysrv")
+	if *showVersion {
+		fmt.Println(longVer)
+		return
+	}
+
 	if extAddress == "" {
 		extAddress = listen
 	}
@@ -146,7 +136,7 @@ func main() {
 		}
 	}
 
-	log.Println(LongVersion)
+	log.Println(longVer)
 
 	maxDescriptors, err := osutil.MaximizeOpenFileLimit()
 	if maxDescriptors > 0 {
@@ -166,7 +156,7 @@ func main() {
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 3072)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
@@ -194,7 +184,7 @@ func main() {
 		log.Println("ID:", id)
 	}
 
-	wrapper := config.Wrap("config", config.New(id))
+	wrapper := config.Wrap("config", config.New(id), events.NoopLogger)
 	wrapper.SetOptions(config.OptionsConfiguration{
 		NATLeaseM:   natLease,
 		NATRenewalM: natRenewal,

cmd/strelaysrv/pool.go

@@ -7,10 +7,15 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"net/url"
 	"time"
 )
 
+const (
+	httpStatusEnhanceYourCalm = 429
+)
+
 func poolHandler(pool string, uri *url.URL, mapping mapping) {
 	if debug {
 		log.Println("Joining", pool)
@@ -28,38 +33,62 @@ func poolHandler(pool string, uri *url.URL, mapping mapping) {
 
 		resp, err := httpClient.Post(pool, "application/json", &b)
 		if err != nil {
-			log.Println("Error joining pool", pool, err)
+			log.Printf("Error joining pool %v: HTTP request: %v", pool, err)
-		} else if resp.StatusCode == 500 {
-			bs, err := ioutil.ReadAll(resp.Body)
-			if err != nil {
-				log.Println("Failed to join", pool, "due to an internal server error. Could not read response:", err)
-			} else {
-				log.Println("Failed to join", pool, "due to an internal server error:", string(bs))
-			}
-			resp.Body.Close()
-		} else if resp.StatusCode == 429 {
-			log.Println(pool, "under load, will retry in a minute")
 			time.Sleep(time.Minute)
 			continue
-		} else if resp.StatusCode == 401 {
+		}
-			log.Println(pool, "failed to join due to IP address not matching external address. Aborting")
+
-			return
+		bs, err := ioutil.ReadAll(resp.Body)
-		} else if resp.StatusCode == 200 {
+		resp.Body.Close()
+		if err != nil {
+			log.Printf("Error joining pool %v: reading response: %v", pool, err)
+			time.Sleep(time.Minute)
+			continue
+		}
+
+		switch resp.StatusCode {
+		case http.StatusOK:
 			var x struct {
 				EvictionIn time.Duration `json:"evictionIn"`
 			}
-			err := json.NewDecoder(resp.Body).Decode(&x)
+			if err := json.NewDecoder(resp.Body).Decode(&x); err == nil {
-			if err == nil {
 				rejoin := x.EvictionIn - (x.EvictionIn / 5)
-				log.Println("Joined", pool, "rejoining in", rejoin)
+				log.Printf("Joined pool %s, rejoining in %v", pool, rejoin)
 				time.Sleep(rejoin)
 				continue
 			} else {
-				log.Println("Failed to deserialize response", err)
+				log.Printf("Joined pool %s, failed to deserialize response: %v", pool, err)
 			}
-		} else {
+
-			log.Println(pool, "unknown response type from server", resp.StatusCode)
+		case http.StatusInternalServerError:
+			log.Printf("Failed to join %v: server error", pool)
+			log.Printf("Response data: %s", bs)
+			time.Sleep(time.Minute)
+			continue
+
+		case http.StatusBadRequest:
+			log.Printf("Failed to join %v: request or check error", pool)
+			log.Printf("Response data: %s", bs)
+			time.Sleep(time.Minute)
+			continue
+
+		case httpStatusEnhanceYourCalm:
+			log.Printf("Failed to join %v: under load (rate limiting)", pool)
+			time.Sleep(time.Minute)
+			continue
+
+		case http.StatusUnauthorized:
+			log.Printf("Failed to join %v: IP address not matching external address", pool)
+			log.Println("Aborting")
+			return
+
+		default:
+			log.Printf("Failed to join %v: unexpected status code from server: %d", pool, resp.StatusCode)
+			log.Printf("Response data: %s", bs)
+			time.Sleep(time.Minute)
+			continue
 		}
+
 		time.Sleep(time.Hour)
 	}
 }

cmd/strelaysrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-relaysrv --ingroup syncthing syncthing-relaysrv

cmd/strelaysrv/session.go

@@ -22,7 +22,7 @@ import (
 var (
 	sessionMut      = sync.RWMutex{}
 	activeSessions  = make([]*session, 0)
-	pendingSessions = make(map[string]*session, 0)
+	pendingSessions = make(map[string]*session)
 	numProxies      int64
 	bytesProxied    int64
 )

cmd/strelaysrv/status.go

@@ -10,6 +10,8 @@ import (
 	"runtime"
 	"sync/atomic"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/build"
 )
 
 var rc *rateCalculator
@@ -40,10 +42,10 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 
 	sessionMut.Lock()
 	// This can potentially be double the number of pending sessions, as each session has two keys, one for each side.
-	status["version"] = Version
+	status["version"] = build.Version
-	status["buildHost"] = BuildHost
+	status["buildHost"] = build.Host
-	status["buildUser"] = BuildUser
+	status["buildUser"] = build.User
-	status["buildDate"] = BuildDate
+	status["buildDate"] = build.Date
 	status["startTime"] = rc.startTime
 	status["uptimeSeconds"] = time.Since(rc.startTime) / time.Second
 	status["numPendingSessionKeys"] = len(pendingSessions)
@@ -86,9 +88,9 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 }
 
 type rateCalculator struct {
+	counter   *int64 // atomic, must remain 64-bit aligned
 	rates     []int64
 	prev      int64
-	counter   *int64
 	startTime time.Time
 }
 

cmd/strelaysrv/testutil/main.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"crypto/tls"
 	"flag"
 	"log"
@@ -19,6 +20,8 @@ import (
 )
 
 func main() {
+	ctx := context.Background()
+
 	log.SetOutput(os.Stdout)
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
 
@@ -76,7 +79,7 @@ func main() {
 		}()
 
 		for {
-			conn, err := client.JoinSession(<-recv)
+			conn, err := client.JoinSession(ctx, <-recv)
 			if err != nil {
 				log.Fatalln("Failed to join", err)
 			}
@@ -90,13 +93,13 @@ func main() {
 			log.Fatal(err)
 		}
 
-		invite, err := client.GetInvitationFromRelay(uri, id, []tls.Certificate{cert}, 10*time.Second)
+		invite, err := client.GetInvitationFromRelay(ctx, uri, id, []tls.Certificate{cert}, 10*time.Second)
 		if err != nil {
 			log.Fatal(err)
 		}
 
 		log.Println("Received invitation", invite)
-		conn, err := client.JoinSession(invite)
+		conn, err := client.JoinSession(ctx, invite)
 		if err != nil {
 			log.Fatalln("Failed to join", err)
 		}
@@ -104,10 +107,10 @@ func main() {
 		connectToStdio(stdin, conn)
 		log.Println("Finished", conn.RemoteAddr(), conn.LocalAddr())
 	} else if test {
-		if client.TestRelay(uri, []tls.Certificate{cert}, time.Second, 2*time.Second, 4) {
+		if err := client.TestRelay(ctx, uri, []tls.Certificate{cert}, time.Second, 2*time.Second, 4); err == nil {
 			log.Println("OK")
 		} else {
-			log.Println("FAIL")
+			log.Println("FAIL:", err)
 		}
 	} else {
 		log.Fatal("Requires either join or connect")

cmd/stupgrades/main.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"os"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/upgrade"
+)
+
+const defaultURL = "https://api.github.com/repos/syncthing/syncthing/releases?per_page=25"
+
+func main() {
+	url := flag.String("u", defaultURL, "GitHub releases url")
+	flag.Parse()
+
+	rels := upgrade.FetchLatestReleases(*url, "")
+	if rels == nil {
+		// An error was already logged
+		os.Exit(1)
+	}
+
+	sort.Sort(upgrade.SortByRelease(rels))
+	rels = filterForLatest(rels)
+
+	if err := json.NewEncoder(os.Stdout).Encode(rels); err != nil {
+		os.Exit(1)
+	}
+}
+
+// filterForLatest returns the latest stable and prerelease only. If the
+// stable version is newer (comes first in the list) there is no need to go
+// looking for a prerelease at all.
+func filterForLatest(rels []upgrade.Release) []upgrade.Release {
+	var filtered []upgrade.Release
+	var havePre bool
+	for _, rel := range rels {
+		if !rel.Prerelease {
+			// We found a stable version, we're good now.
+			filtered = append(filtered, rel)
+			break
+		}
+		if rel.Prerelease && !havePre {
+			// We remember the first prerelease we find.
+			filtered = append(filtered, rel)
+			havePre = true
+		}
+	}
+	return filtered
+}

cmd/stvanity/main.go

@@ -14,6 +14,7 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
+	"errors"
 	"flag"
 	"fmt"
 	"math/big"
@@ -191,7 +192,7 @@ func pemBlockForKey(priv interface{}) (*pem.Block, error) {
 		}
 		return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}, nil
 	default:
-		return nil, fmt.Errorf("unknown key type")
+		return nil, errors.New("unknown key type")
 	}
 }
 

cmd/syncthing/auditservice.go

@@ -1,69 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"encoding/json"
-	"io"
-
-	"github.com/syncthing/syncthing/lib/events"
-)
-
-// The auditService subscribes to events and writes these in JSON format, one
-// event per line, to the specified writer.
-type auditService struct {
-	w       io.Writer     // audit destination
-	stop    chan struct{} // signals time to stop
-	started chan struct{} // signals startup complete
-	stopped chan struct{} // signals stop complete
-}
-
-func newAuditService(w io.Writer) *auditService {
-	return &auditService{
-		w:       w,
-		stop:    make(chan struct{}),
-		started: make(chan struct{}),
-		stopped: make(chan struct{}),
-	}
-}
-
-// Serve runs the audit service.
-func (s *auditService) Serve() {
-	defer close(s.stopped)
-	sub := events.Default.Subscribe(events.AllEvents)
-	defer events.Default.Unsubscribe(sub)
-	enc := json.NewEncoder(s.w)
-
-	// We're ready to start processing events.
-	close(s.started)
-
-	for {
-		select {
-		case ev := <-sub.C():
-			enc.Encode(ev)
-		case <-s.stop:
-			return
-		}
-	}
-}
-
-// Stop stops the audit service.
-func (s *auditService) Stop() {
-	close(s.stop)
-}
-
-// WaitForStart returns once the audit service is ready to receive events, or
-// immediately if it's already running.
-func (s *auditService) WaitForStart() {
-	<-s.started
-}
-
-// WaitForStop returns once the audit service has stopped.
-// (Needed by the tests.)
-func (s *auditService) WaitForStop() {
-	<-s.stopped
-}

cmd/syncthing/blockprof.go

@@ -22,11 +22,15 @@ func init() {
 			panic("Couldn't find block profiler")
 		}
 		l.Debugln("Starting block profiling")
-		go saveBlockingProfiles(profiler)
+		go func() {
+			err := saveBlockingProfiles(profiler) // Only returns on error
+			l.Warnln("Block profiler failed:", err)
+			panic("Block profiler failed")
+		}()
 	}
 }
 
-func saveBlockingProfiles(profiler *pprof.Profile) {
+func saveBlockingProfiles(profiler *pprof.Profile) error {
 	runtime.SetBlockProfileRate(1)
 
 	t0 := time.Now()
@@ -35,16 +39,16 @@ func saveBlockingProfiles(profiler *pprof.Profile) {
 
 		fd, err := os.Create(fmt.Sprintf("block-%05d-%07d.pprof", syscall.Getpid(), startms))
 		if err != nil {
-			panic(err)
+			return err
 		}
 		err = profiler.WriteTo(fd, 0)
 		if err != nil {
-			panic(err)
+			return err
 		}
 		err = fd.Close()
 		if err != nil {
-			panic(err)
+			return err
 		}
-
 	}
+	return nil
 }

cmd/syncthing/cpuusage.go

@@ -1,59 +0,0 @@
-// Copyright (C) 2017 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"math"
-	"time"
-
-	metrics "github.com/rcrowley/go-metrics"
-)
-
-const cpuTickRate = 5 * time.Second
-
-type cpuService struct {
-	avg  metrics.EWMA
-	stop chan struct{}
-}
-
-func newCPUService() *cpuService {
-	return &cpuService{
-		// 10 second average. Magic alpha value comes from looking at EWMA package
-		// definitions of EWMA1, EWMA5. The tick rate *must* be five seconds (hard
-		// coded in the EWMA package).
-		avg:  metrics.NewEWMA(1 - math.Exp(-float64(cpuTickRate)/float64(time.Second)/10.0)),
-		stop: make(chan struct{}),
-	}
-}
-
-func (s *cpuService) Serve() {
-	// Initialize prevUsage to an actual value returned by cpuUsage
-	// instead of zero, because at least Windows returns a huge negative
-	// number here that then slowly increments...
-	prevUsage := cpuUsage()
-	ticker := time.NewTicker(cpuTickRate)
-	defer ticker.Stop()
-	for {
-		select {
-		case <-ticker.C:
-			curUsage := cpuUsage()
-			s.avg.Update(int64((curUsage - prevUsage) / time.Millisecond))
-			prevUsage = curUsage
-			s.avg.Tick()
-		case <-s.stop:
-			return
-		}
-	}
-}
-
-func (s *cpuService) Stop() {
-	close(s.stop)
-}
-
-func (s *cpuService) Rate() float64 {
-	return s.avg.Rate()
-}

cmd/syncthing/cpuusage_solaris.go

@@ -1,78 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//+build solaris
-
-package main
-
-import (
-	"encoding/binary"
-	"fmt"
-	"os"
-	"time"
-)
-
-type id_t int32
-type ulong_t uint32
-
-type timestruc_t struct {
-	Tv_sec  int64
-	Tv_nsec int64
-}
-
-func (tv timestruc_t) Nano() int64 {
-	return tv.Tv_sec*1e9 + tv.Tv_nsec
-}
-
-type prusage_t struct {
-	Pr_lwpid    id_t        /* lwp id.  0: process or defunct */
-	Pr_count    int32       /* number of contributing lwps */
-	Pr_tstamp   timestruc_t /* real time stamp, time of read() */
-	Pr_create   timestruc_t /* process/lwp creation time stamp */
-	Pr_term     timestruc_t /* process/lwp termination time stamp */
-	Pr_rtime    timestruc_t /* total lwp real (elapsed) time */
-	Pr_utime    timestruc_t /* user level CPU time */
-	Pr_stime    timestruc_t /* system call CPU time */
-	Pr_ttime    timestruc_t /* other system trap CPU time */
-	Pr_tftime   timestruc_t /* text page fault sleep time */
-	Pr_dftime   timestruc_t /* data page fault sleep time */
-	Pr_kftime   timestruc_t /* kernel page fault sleep time */
-	Pr_ltime    timestruc_t /* user lock wait sleep time */
-	Pr_slptime  timestruc_t /* all other sleep time */
-	Pr_wtime    timestruc_t /* wait-cpu (latency) time */
-	Pr_stoptime timestruc_t /* stopped time */
-	Pr_minf     ulong_t     /* minor page faults */
-	Pr_majf     ulong_t     /* major page faults */
-	Pr_nswap    ulong_t     /* swaps */
-	Pr_inblk    ulong_t     /* input blocks */
-	Pr_oublk    ulong_t     /* output blocks */
-	Pr_msnd     ulong_t     /* messages sent */
-	Pr_mrcv     ulong_t     /* messages received */
-	Pr_sigs     ulong_t     /* signals received */
-	Pr_vctx     ulong_t     /* voluntary context switches */
-	Pr_ictx     ulong_t     /* involuntary context switches */
-	Pr_sysc     ulong_t     /* system calls */
-	Pr_ioch     ulong_t     /* chars read and written */
-
-}
-
-var procFile = fmt.Sprintf("/proc/%d/usage", os.Getpid())
-
-func cpuUsage() time.Duration {
-	fd, err := os.Open(procFile)
-	if err != nil {
-		return 0
-	}
-
-	var rusage prusage_t
-	err = binary.Read(fd, binary.LittleEndian, rusage)
-	fd.Close()
-	if err != nil {
-		return 0
-	}
-
-	return time.Duration(rusage.Pr_utime.Nano() + rusage.Pr_stime.Nano())
-}

cmd/syncthing/cpuusage_windows.go

@@ -1,27 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//+build windows
-
-package main
-
-import "syscall"
-import "time"
-
-func cpuUsage() time.Duration {
-	handle, err := syscall.GetCurrentProcess()
-	if err != nil {
-		return 0
-	}
-	defer syscall.CloseHandle(handle)
-
-	var ctime, etime, ktime, utime syscall.Filetime
-	if err := syscall.GetProcessTimes(handle, &ctime, &etime, &ktime, &utime); err != nil {
-		return 0
-	}
-
-	return time.Duration(ktime.Nanoseconds() + utime.Nanoseconds())
-}

cmd/syncthing/crash_reporting.go

@@ -0,0 +1,152 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/sha256"
+)
+
+const (
+	headRequestTimeout = 10 * time.Second
+	putRequestTimeout  = time.Minute
+)
+
+// uploadPanicLogs attempts to upload all the panic logs in the named
+// directory to the crash reporting server as urlBase. Uploads are attempted
+// with the newest log first.
+//
+// This can can block for a long time. The context can set a final deadline
+// for this.
+func uploadPanicLogs(ctx context.Context, urlBase, dir string) {
+	files, err := filepath.Glob(filepath.Join(dir, "panic-*.log"))
+	if err != nil {
+		l.Warnln("Failed to list panic logs:", err)
+		return
+	}
+
+	sort.Sort(sort.Reverse(sort.StringSlice(files)))
+	for _, file := range files {
+		if strings.Contains(file, ".reported.") {
+			// We've already sent this file. It'll be cleaned out at some
+			// point.
+			continue
+		}
+
+		if err := uploadPanicLog(ctx, urlBase, file); err != nil {
+			l.Warnln("Reporting crash:", err)
+		} else {
+			// Rename the log so we don't have to try to report it again. This
+			// succeeds, or it does not. There is no point complaining about it.
+			_ = os.Rename(file, strings.Replace(file, ".log", ".reported.log", 1))
+		}
+	}
+}
+
+// uploadPanicLog attempts to upload the named panic log to the crash
+// reporting server at urlBase. The panic ID is constructed as the sha256 of
+// the log contents. A HEAD request is made to see if the log has already
+// been reported. If not, a PUT is made with the log contents.
+func uploadPanicLog(ctx context.Context, urlBase, file string) error {
+	data, err := ioutil.ReadFile(file)
+	if err != nil {
+		return err
+	}
+
+	// Remove log lines, for privacy.
+	data = filterLogLines(data)
+
+	hash := fmt.Sprintf("%x", sha256.Sum256(data))
+	l.Infof("Reporting crash found in %s (report ID %s) ...\n", filepath.Base(file), hash[:8])
+
+	url := fmt.Sprintf("%s/%s", urlBase, hash)
+	headReq, err := http.NewRequest(http.MethodHead, url, nil)
+	if err != nil {
+		return err
+	}
+
+	// Set a reasonable timeout on the HEAD request
+	headCtx, headCancel := context.WithTimeout(ctx, headRequestTimeout)
+	defer headCancel()
+	headReq = headReq.WithContext(headCtx)
+
+	resp, err := http.DefaultClient.Do(headReq)
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	if resp.StatusCode == http.StatusOK {
+		// It's known, we're done
+		return nil
+	}
+
+	putReq, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+
+	// Set a reasonable timeout on the PUT request
+	putCtx, putCancel := context.WithTimeout(ctx, putRequestTimeout)
+	defer putCancel()
+	putReq = putReq.WithContext(putCtx)
+
+	resp, err = http.DefaultClient.Do(putReq)
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("upload: %s", resp.Status)
+	}
+
+	return nil
+}
+
+// filterLogLines returns the data without any log lines between the first
+// line and the panic trace. This is done in-place: the original data slice
+// is destroyed.
+func filterLogLines(data []byte) []byte {
+	filtered := data[:0]
+	matched := false
+	for _, line := range bytes.Split(data, []byte("\n")) {
+		switch {
+		case !matched && bytes.HasPrefix(line, []byte("Panic ")):
+			// This begins the panic trace, set the matched flag and append.
+			matched = true
+			fallthrough
+		case len(filtered) == 0 || matched:
+			// This is the first line or inside the panic trace.
+			if len(filtered) > 0 {
+				// We add the newline before rather than after because
+				// bytes.Split sees the \n as *separator* and not line
+				// ender, so ir will generate a last empty line that we
+				// don't really want. (We want to keep blank lines in the
+				// middle of the trace though.)
+				filtered = append(filtered, '\n')
+			}
+			// Remove the device ID prefix. The "plus two" stuff is because
+			// the line will look like "[foo] whatever" and the end variable
+			// will end up pointing at the ] and we want to step over that
+			// and the following space.
+			if end := bytes.Index(line, []byte("]")); end > 1 && end < len(line)-2 && bytes.HasPrefix(line, []byte("[")) {
+				line = line[end+2:]
+			}
+			filtered = append(filtered, line...)
+		}
+	}
+	return filtered
+}

cmd/syncthing/crash_reporting_test.go

@@ -0,0 +1,37 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestFilterLogLines(t *testing.T) {
+	in := []byte(`[ABCD123] syncthing version whatever
+here is more log data
+and more
+...
+and some more
+yet more
+Panic detected at like right now
+here is panic data
+and yet more panic stuff
+`)
+
+	filtered := []byte(`syncthing version whatever
+Panic detected at like right now
+here is panic data
+and yet more panic stuff
+`)
+
+	result := filterLogLines(in)
+	if !bytes.Equal(result, filtered) {
+		t.Logf("%q\n", result)
+		t.Error("it should have been filtered")
+	}
+}

cmd/syncthing/debug.go

@@ -7,22 +7,9 @@
 package main
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
-	l     = logger.DefaultLogger.NewFacility("main", "Main package")
+	l = logger.DefaultLogger.NewFacility("main", "Main package")
-	httpl = logger.DefaultLogger.NewFacility("http", "REST API")
 )
-
-func shouldDebugHTTP() bool {
-	return l.ShouldDebug("http")
-}
-
-func init() {
-	l.SetDebug("main", strings.Contains(os.Getenv("STTRACE"), "main") || os.Getenv("STTRACE") == "all")
-	l.SetDebug("http", strings.Contains(os.Getenv("STTRACE"), "http") || os.Getenv("STTRACE") == "all")
-}

cmd/syncthing/gui_csrf.go

@@ -1,142 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bufio"
-	"fmt"
-	"net/http"
-	"os"
-	"strings"
-
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/osutil"
-	"github.com/syncthing/syncthing/lib/rand"
-	"github.com/syncthing/syncthing/lib/sync"
-)
-
-// csrfTokens is a list of valid tokens. It is sorted so that the most
-// recently used token is first in the list. New tokens are added to the front
-// of the list (as it is the most recently used at that time). The list is
-// pruned to a maximum of maxCsrfTokens, throwing away the least recently used
-// tokens.
-var csrfTokens []string
-var csrfMut = sync.NewMutex()
-
-const maxCsrfTokens = 25
-
-// Check for CSRF token on /rest/ URLs. If a correct one is not given, reject
-// the request with 403. For / and /index.html, set a new CSRF cookie if none
-// is currently set.
-func csrfMiddleware(unique string, prefix string, cfg config.GUIConfiguration, next http.Handler) http.Handler {
-	loadCsrfTokens()
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// Allow requests carrying a valid API key
-		if cfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
-			// Set the access-control-allow-origin header for CORS requests
-			// since a valid API key has been provided
-			w.Header().Add("Access-Control-Allow-Origin", "*")
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		if strings.HasPrefix(r.URL.Path, "/rest/debug") {
-			// Debugging functions are only available when explicitly
-			// enabled, and can be accessed without a CSRF token
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		// Allow requests for anything not under the protected path prefix,
-		// and set a CSRF cookie if there isn't already a valid one.
-		if !strings.HasPrefix(r.URL.Path, prefix) {
-			cookie, err := r.Cookie("CSRF-Token-" + unique)
-			if err != nil || !validCsrfToken(cookie.Value) {
-				httpl.Debugln("new CSRF cookie in response to request for", r.URL)
-				cookie = &http.Cookie{
-					Name:  "CSRF-Token-" + unique,
-					Value: newCsrfToken(),
-				}
-				http.SetCookie(w, cookie)
-			}
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		// Verify the CSRF token
-		token := r.Header.Get("X-CSRF-Token-" + unique)
-		if !validCsrfToken(token) {
-			http.Error(w, "CSRF Error", 403)
-			return
-		}
-
-		next.ServeHTTP(w, r)
-	})
-}
-
-func validCsrfToken(token string) bool {
-	csrfMut.Lock()
-	defer csrfMut.Unlock()
-	for i, t := range csrfTokens {
-		if t == token {
-			if i > 0 {
-				// Move this token to the head of the list. Copy the tokens at
-				// the front one step to the right and then replace the token
-				// at the head.
-				copy(csrfTokens[1:], csrfTokens[:i+1])
-				csrfTokens[0] = token
-			}
-			return true
-		}
-	}
-	return false
-}
-
-func newCsrfToken() string {
-	token := rand.String(32)
-
-	csrfMut.Lock()
-	csrfTokens = append([]string{token}, csrfTokens...)
-	if len(csrfTokens) > maxCsrfTokens {
-		csrfTokens = csrfTokens[:maxCsrfTokens]
-	}
-	defer csrfMut.Unlock()
-
-	saveCsrfTokens()
-
-	return token
-}
-
-func saveCsrfTokens() {
-	// We're ignoring errors in here. It's not super critical and there's
-	// nothing relevant we can do about them anyway...
-
-	name := locations[locCsrfTokens]
-	f, err := osutil.CreateAtomic(name)
-	if err != nil {
-		return
-	}
-
-	for _, t := range csrfTokens {
-		fmt.Fprintln(f, t)
-	}
-
-	f.Close()
-}
-
-func loadCsrfTokens() {
-	f, err := os.Open(locations[locCsrfTokens])
-	if err != nil {
-		return
-	}
-	defer f.Close()
-
-	s := bufio.NewScanner(f)
-	for s.Scan() {
-		csrfTokens = append(csrfTokens, s.Text())
-	}
-}

cmd/syncthing/heapprof.go

@@ -23,11 +23,15 @@ func init() {
 			rate = i
 		}
 		l.Debugln("Starting heap profiling")
-		go saveHeapProfiles(rate)
+		go func() {
+			err := saveHeapProfiles(rate) // Only returns on error
+			l.Warnln("Heap profiler failed:", err)
+			panic("Heap profiler failed")
+		}()
 	}
 }
 
-func saveHeapProfiles(rate int) {
+func saveHeapProfiles(rate int) error {
 	runtime.MemProfileRate = rate
 	var memstats, prevMemstats runtime.MemStats
 
@@ -38,21 +42,21 @@ func saveHeapProfiles(rate int) {
 		if memstats.HeapInuse > prevMemstats.HeapInuse {
 			fd, err := os.Create(name + ".tmp")
 			if err != nil {
-				panic(err)
+				return err
 			}
 			err = pprof.WriteHeapProfile(fd)
 			if err != nil {
-				panic(err)
+				return err
 			}
 			err = fd.Close()
 			if err != nil {
-				panic(err)
+				return err
 			}
 
-			_ = os.Remove(name) // Error deliberately ignored
+			os.Remove(name) // Error deliberately ignored
 			err = os.Rename(name+".tmp", name)
 			if err != nil {
-				panic(err)
+				return err
 			}
 
 			prevMemstats = memstats

cmd/syncthing/locations.go

@@ -1,125 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"os"
-	"path/filepath"
-	"runtime"
-	"strings"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/fs"
-)
-
-type locationEnum string
-
-// Use strings as keys to make printout and serialization of the locations map
-// more meaningful.
-const (
-	locConfigFile    locationEnum = "config"
-	locCertFile                   = "certFile"
-	locKeyFile                    = "keyFile"
-	locHTTPSCertFile              = "httpsCertFile"
-	locHTTPSKeyFile               = "httpsKeyFile"
-	locDatabase                   = "database"
-	locLogFile                    = "logFile"
-	locCsrfTokens                 = "csrfTokens"
-	locPanicLog                   = "panicLog"
-	locAuditLog                   = "auditLog"
-	locGUIAssets                  = "GUIAssets"
-	locDefFolder                  = "defFolder"
-)
-
-// Platform dependent directories
-var baseDirs = map[string]string{
-	"config": defaultConfigDir(), // Overridden by -home flag
-	"home":   homeDir(),          // User's home directory, *not* -home flag
-}
-
-// Use the variables from baseDirs here
-var locations = map[locationEnum]string{
-	locConfigFile:    "${config}/config.xml",
-	locCertFile:      "${config}/cert.pem",
-	locKeyFile:       "${config}/key.pem",
-	locHTTPSCertFile: "${config}/https-cert.pem",
-	locHTTPSKeyFile:  "${config}/https-key.pem",
-	locDatabase:      "${config}/index-v0.14.0.db",
-	locLogFile:       "${config}/syncthing.log", // -logfile on Windows
-	locCsrfTokens:    "${config}/csrftokens.txt",
-	locPanicLog:      "${config}/panic-${timestamp}.log",
-	locAuditLog:      "${config}/audit-${timestamp}.log",
-	locGUIAssets:     "${config}/gui",
-	locDefFolder:     "${home}/Sync",
-}
-
-// expandLocations replaces the variables in the location map with actual
-// directory locations.
-func expandLocations() error {
-	for key, dir := range locations {
-		for varName, value := range baseDirs {
-			dir = strings.Replace(dir, "${"+varName+"}", value, -1)
-		}
-		var err error
-		dir, err = fs.ExpandTilde(dir)
-		if err != nil {
-			return err
-		}
-		locations[key] = dir
-	}
-	return nil
-}
-
-// defaultConfigDir returns the default configuration directory, as figured
-// out by various the environment variables present on each platform, or dies
-// trying.
-func defaultConfigDir() string {
-	switch runtime.GOOS {
-	case "windows":
-		if p := os.Getenv("LocalAppData"); p != "" {
-			return filepath.Join(p, "Syncthing")
-		}
-		return filepath.Join(os.Getenv("AppData"), "Syncthing")
-
-	case "darwin":
-		dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
-		if err != nil {
-			l.Fatalln(err)
-		}
-		return dir
-
-	default:
-		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
-			return filepath.Join(xdgCfg, "syncthing")
-		}
-		dir, err := fs.ExpandTilde("~/.config/syncthing")
-		if err != nil {
-			l.Fatalln(err)
-		}
-		return dir
-	}
-}
-
-// homeDir returns the user's home directory, or dies trying.
-func homeDir() string {
-	home, err := fs.ExpandTilde("~")
-	if err != nil {
-		l.Fatalln(err)
-	}
-	return home
-}
-
-func timestampedLoc(key locationEnum) string {
-	// We take the roundtrip via "${timestamp}" instead of passing the path
-	// directly through time.Format() to avoid issues when the path we are
-	// expanding contains numbers; otherwise for example
-	// /home/user2006/.../panic-20060102-150405.log would get both instances of
-	// 2006 replaced by 2015...
-	tpl := locations[key]
-	now := time.Now().Format("20060102-150405")
-	return strings.Replace(tpl, "${timestamp}", now, -1)
-}

cmd/syncthing/memsize_darwin.go

@@ -1,31 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"errors"
-	"os/exec"
-	"strconv"
-	"strings"
-)
-
-func memorySize() (int64, error) {
-	cmd := exec.Command("sysctl", "hw.memsize")
-	out, err := cmd.Output()
-	if err != nil {
-		return 0, err
-	}
-	fs := strings.Fields(string(out))
-	if len(fs) != 2 {
-		return 0, errors.New("sysctl parse error")
-	}
-	bytes, err := strconv.ParseInt(fs[1], 10, 64)
-	if err != nil {
-		return 0, err
-	}
-	return bytes, nil
-}

cmd/syncthing/mocked_config_test.go

@@ -1,69 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/util"
-)
-
-type mockedConfig struct {
-	gui config.GUIConfiguration
-}
-
-func (c *mockedConfig) GUI() config.GUIConfiguration {
-	return c.gui
-}
-
-func (c *mockedConfig) ListenAddresses() []string {
-	return nil
-}
-
-func (c *mockedConfig) LDAP() config.LDAPConfiguration {
-	return config.LDAPConfiguration{}
-}
-
-func (c *mockedConfig) RawCopy() config.Configuration {
-	cfg := config.Configuration{}
-	util.SetDefaults(&cfg.Options)
-	return cfg
-}
-
-func (c *mockedConfig) Options() config.OptionsConfiguration {
-	return config.OptionsConfiguration{}
-}
-
-func (c *mockedConfig) Replace(cfg config.Configuration) (config.Waiter, error) {
-	return nil, nil
-}
-
-func (c *mockedConfig) Subscribe(cm config.Committer) {}
-
-func (c *mockedConfig) Folders() map[string]config.FolderConfiguration {
-	return nil
-}
-
-func (c *mockedConfig) Devices() map[protocol.DeviceID]config.DeviceConfiguration {
-	return nil
-}
-
-func (c *mockedConfig) SetDevice(config.DeviceConfiguration) (config.Waiter, error) {
-	return nil, nil
-}
-
-func (c *mockedConfig) SetDevices([]config.DeviceConfiguration) (config.Waiter, error) {
-	return nil, nil
-}
-
-func (c *mockedConfig) Save() error {
-	return nil
-}
-
-func (c *mockedConfig) RequiresRestart() bool {
-	return false
-}

cmd/syncthing/mocked_connections_test.go

@@ -1,17 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-type mockedConnections struct{}
-
-func (m *mockedConnections) Status() map[string]interface{} {
-	return nil
-}
-
-func (m *mockedConnections) NATType() string {
-	return ""
-}

cmd/syncthing/monitor.go

@@ -8,17 +8,25 @@ package main
 
 import (
 	"bufio"
+	"context"
+	"fmt"
 	"io"
 	"os"
 	"os/exec"
 	"os/signal"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/fs"
+	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/syncthing"
 )
 
 var (
@@ -32,6 +40,8 @@ const (
 	loopThreshold         = 60 * time.Second
 	logFileAutoCloseDelay = 5 * time.Second
 	logFileMaxOpenTime    = time.Minute
+	panicUploadMaxWait    = 30 * time.Second
+	panicUploadNoticeWait = 10 * time.Second
 )
 
 func monitorMain(runtimeOptions RuntimeOptions) {
@@ -41,28 +51,42 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 
 	logFile := runtimeOptions.logFile
 	if logFile != "-" {
-		var fileDst io.Writer = newAutoclosedFile(logFile, logFileAutoCloseDelay, logFileMaxOpenTime)
+		if expanded, err := fs.ExpandTilde(logFile); err == nil {
-
+			logFile = expanded
-		if runtime.GOOS == "windows" {
-			// Translate line breaks to Windows standard
-			fileDst = osutil.ReplacingWriter{
-				Writer: fileDst,
-				From:   '\n',
-				To:     []byte{'\r', '\n'},
-			}
 		}
+		var fileDst io.Writer
+		var err error
+		open := func(name string) (io.WriteCloser, error) {
+			return newAutoclosedFile(name, logFileAutoCloseDelay, logFileMaxOpenTime)
+		}
+		if runtimeOptions.logMaxSize > 0 {
+			fileDst, err = newRotatedFile(logFile, open, int64(runtimeOptions.logMaxSize), runtimeOptions.logMaxFiles)
+		} else {
+			fileDst, err = open(logFile)
+		}
+		if err != nil {
+			l.Warnln("Failed to setup logging to file, proceeding with logging to stdout only:", err)
+		} else {
+			if runtime.GOOS == "windows" {
+				// Translate line breaks to Windows standard
+				fileDst = osutil.ReplacingWriter{
+					Writer: fileDst,
+					From:   '\n',
+					To:     []byte{'\r', '\n'},
+				}
+			}
 
-		// Log to both stdout and file.
+			// Log to both stdout and file.
-		dst = io.MultiWriter(dst, fileDst)
+			dst = io.MultiWriter(dst, fileDst)
 
-		l.Infof(`Log output saved to file "%s"`, logFile)
+			l.Infof(`Log output saved to file "%s"`, logFile)
+		}
 	}
 
 	args := os.Args
 	var restarts [countRestarts]time.Time
 
 	stopSign := make(chan os.Signal, 1)
-	sigTerm := syscall.Signal(15)
 	signal.Notify(stopSign, os.Interrupt, sigTerm)
 	restartSign := make(chan os.Signal, 1)
 	sigHup := syscall.Signal(1)
@@ -71,9 +95,11 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 	childEnv := childEnv()
 	first := true
 	for {
+		maybeReportPanics()
+
 		if t := time.Since(restarts[0]); t < loopThreshold {
 			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 
 		copy(restarts[0:], restarts[1:])
@@ -84,18 +110,19 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 
 		stderr, err := cmd.StderrPipe()
 		if err != nil {
-			l.Fatalln("stderr:", err)
+			panic(err)
 		}
 
 		stdout, err := cmd.StdoutPipe()
 		if err != nil {
-			l.Fatalln("stdout:", err)
+			panic(err)
 		}
 
-		l.Infoln("Starting syncthing")
+		l.Debugln("Starting syncthing")
 		err = cmd.Start()
 		if err != nil {
-			l.Fatalln(err)
+			l.Warnln("Error starting the main Syncthing process:", err)
+			panic("Error starting the main Syncthing process")
 		}
 
 		stdoutMut.Lock()
@@ -124,12 +151,13 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 			exit <- cmd.Wait()
 		}()
 
+		stopped := false
 		select {
 		case s := <-stopSign:
 			l.Infof("Signal %d received; exiting", s)
-			cmd.Process.Kill()
+			cmd.Process.Signal(sigTerm)
-			<-exit
+			err = <-exit
-			return
+			stopped = true
 
 		case s := <-restartSign:
 			l.Infof("Signal %d received; restarting", s)
@@ -137,23 +165,31 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 			err = <-exit
 
 		case err = <-exit:
-			if err == nil {
+		}
-				// Successful exit indicates an intentional shutdown
+
-				return
+		if err == nil {
-			} else if exiterr, ok := err.(*exec.ExitError); ok {
+			// Successful exit indicates an intentional shutdown
-				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+			os.Exit(syncthing.ExitSuccess.AsInt())
-					switch status.ExitStatus() {
+		}
-					case exitUpgrading:
+
-						// Restart the monitor process to release the .old
+		if exiterr, ok := err.(*exec.ExitError); ok {
-						// binary as part of the upgrade process.
+			exitCode := exiterr.ExitCode()
-						l.Infoln("Restarting monitor...")
+			if stopped || runtimeOptions.noRestart {
-						if err = restartMonitor(args); err != nil {
+				os.Exit(exitCode)
-							l.Warnln("Restart:", err)
-						}
-						return
-					}
-				}
 			}
+			if exitCode == syncthing.ExitUpgrade.AsInt() {
+				// Restart the monitor process to release the .old
+				// binary as part of the upgrade process.
+				l.Infoln("Restarting monitor...")
+				if err = restartMonitor(args); err != nil {
+					l.Warnln("Restart:", err)
+				}
+				os.Exit(exitCode)
+			}
+		}
+
+		if runtimeOptions.noRestart {
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 
 		l.Infoln("Syncthing exited:", err)
@@ -172,6 +208,13 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
 	br := bufio.NewReader(stderr)
 
 	var panicFd *os.File
+	defer func() {
+		if panicFd != nil {
+			_ = panicFd.Close()
+			maybeReportPanics()
+		}
+	}()
+
 	for {
 		line, err := br.ReadString('\n')
 		if err != nil {
@@ -198,7 +241,7 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
 			}
 
 			if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
-				panicFd, err = os.Create(timestampedLoc(locPanicLog))
+				panicFd, err = os.Create(locations.GetTimestamped(locations.PanicLog))
 				if err != nil {
 					l.Warnln("Create panic log:", err)
 					continue
@@ -268,6 +311,11 @@ func copyStdout(stdout io.Reader, dst io.Writer) {
 }
 
 func restartMonitor(args []string) error {
+	// Set the STRESTART environment variable to indicate to the next
+	// process that this is a restart and not initial start. This prevents
+	// opening the browser on startup.
+	os.Setenv("STRESTART", "yes")
+
 	if runtime.GOOS != "windows" {
 		// syscall.Exec is the cleanest way to restart on Unixes as it
 		// replaces the current process with the new one, keeping the pid and
@@ -303,6 +351,89 @@ func restartMonitorWindows(args []string) error {
 	return cmd.Start()
 }
 
+// rotatedFile keeps a set of rotating logs. There will be the base file plus up
+// to maxFiles rotated ones, each ~ maxSize bytes large.
+type rotatedFile struct {
+	name        string
+	create      createFn
+	maxSize     int64 // bytes
+	maxFiles    int
+	currentFile io.WriteCloser
+	currentSize int64
+}
+
+type createFn func(name string) (io.WriteCloser, error)
+
+func newRotatedFile(name string, create createFn, maxSize int64, maxFiles int) (*rotatedFile, error) {
+	var size int64
+	if info, err := os.Lstat(name); err != nil {
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+		size = 0
+	} else {
+		size = info.Size()
+	}
+	writer, err := create(name)
+	if err != nil {
+		return nil, err
+	}
+	return &rotatedFile{
+		name:        name,
+		create:      create,
+		maxSize:     maxSize,
+		maxFiles:    maxFiles,
+		currentFile: writer,
+		currentSize: size,
+	}, nil
+}
+
+func (r *rotatedFile) Write(bs []byte) (int, error) {
+	// Check if we're about to exceed the max size, and if so close this
+	// file so we'll start on a new one.
+	if r.currentSize+int64(len(bs)) > r.maxSize {
+		r.currentFile.Close()
+		r.currentSize = 0
+		r.rotate()
+		f, err := r.create(r.name)
+		if err != nil {
+			return 0, err
+		}
+		r.currentFile = f
+	}
+
+	n, err := r.currentFile.Write(bs)
+	r.currentSize += int64(n)
+	return n, err
+}
+
+func (r *rotatedFile) rotate() {
+	// The files are named "name", "name.0", "name.1", ...
+	// "name.(r.maxFiles-1)". Increase the numbers on the
+	// suffixed ones.
+	for i := r.maxFiles - 1; i > 0; i-- {
+		from := numberedFile(r.name, i-1)
+		to := numberedFile(r.name, i)
+		err := os.Rename(from, to)
+		if err != nil && !os.IsNotExist(err) {
+			fmt.Println("LOG: Rotating logs:", err)
+		}
+	}
+
+	// Rename the base to base.0
+	err := os.Rename(r.name, numberedFile(r.name, 0))
+	if err != nil && !os.IsNotExist(err) {
+		fmt.Println("LOG: Rotating logs:", err)
+	}
+}
+
+// numberedFile adds the number between the file name and the extension.
+func numberedFile(name string, num int) string {
+	ext := filepath.Ext(name) // contains the dot
+	withoutExt := name[:len(name)-len(ext)]
+	return fmt.Sprintf("%s.%d%s", withoutExt, num, ext)
+}
+
 // An autoclosedFile is an io.WriteCloser that opens itself for appending on
 // Write() and closes itself after an interval of no writes (closeDelay) or
 // when the file has been open for too long (maxOpenTime). A call to Write()
@@ -321,7 +452,7 @@ type autoclosedFile struct {
 	mut sync.Mutex
 }
 
-func newAutoclosedFile(name string, closeDelay, maxOpenTime time.Duration) *autoclosedFile {
+func newAutoclosedFile(name string, closeDelay, maxOpenTime time.Duration) (*autoclosedFile, error) {
 	f := &autoclosedFile{
 		name:        name,
 		closeDelay:  closeDelay,
@@ -330,8 +461,13 @@ func newAutoclosedFile(name string, closeDelay, maxOpenTime time.Duration) *auto
 		closed:      make(chan struct{}),
 		closeTimer:  time.NewTimer(time.Minute),
 	}
+	f.mut.Lock()
+	defer f.mut.Unlock()
+	if err := f.ensureOpenLocked(); err != nil {
+		return nil, err
+	}
 	go f.closerLoop()
-	return f
+	return f, nil
 }
 
 func (f *autoclosedFile) Write(bs []byte) (int, error) {
@@ -339,7 +475,7 @@ func (f *autoclosedFile) Write(bs []byte) (int, error) {
 	defer f.mut.Unlock()
 
 	// Make sure the file is open for appending
-	if err := f.ensureOpen(); err != nil {
+	if err := f.ensureOpenLocked(); err != nil {
 		return 0, err
 	}
 
@@ -369,22 +505,14 @@ func (f *autoclosedFile) Close() error {
 }
 
 // Must be called with f.mut held!
-func (f *autoclosedFile) ensureOpen() error {
+func (f *autoclosedFile) ensureOpenLocked() error {
 	if f.fd != nil {
 		// File is already open
 		return nil
 	}
 
 	// We open the file for write only, and create it if it doesn't exist.
-	flags := os.O_WRONLY | os.O_CREATE
+	flags := os.O_WRONLY | os.O_CREATE | os.O_APPEND
-	if f.opened.IsZero() {
-		// This is the first time we are opening the file. We should truncate
-		// it to better emulate an os.Create() call.
-		flags |= os.O_TRUNC
-	} else {
-		// The file was already opened once, so we should append to it.
-		flags |= os.O_APPEND
-	}
 
 	fd, err := os.OpenFile(f.name, flags, 0644)
 	if err != nil {
@@ -418,10 +546,10 @@ func (f *autoclosedFile) closerLoop() {
 func childEnv() []string {
 	var env []string
 	for _, str := range os.Environ() {
-		if strings.HasPrefix("STNORESTART=", str) {
+		if strings.HasPrefix(str, "STNORESTART=") {
 			continue
 		}
-		if strings.HasPrefix("STMONITORED=", str) {
+		if strings.HasPrefix(str, "STMONITORED=") {
 			continue
 		}
 		env = append(env, str)
@@ -429,3 +557,39 @@ func childEnv() []string {
 	env = append(env, "STMONITORED=yes")
 	return env
 }
+
+// maybeReportPanics tries to figure out if crash reporting is on or off,
+// and reports any panics it can find if it's enabled. We spend at most
+// panicUploadMaxWait uploading panics...
+func maybeReportPanics() {
+	// Try to get a config to see if/where panics should be reported.
+	cfg, err := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger)
+	if err != nil {
+		l.Warnln("Couldn't load config; not reporting crash")
+		return
+	}
+
+	// Bail if we're not supposed to report panics.
+	opts := cfg.Options()
+	if !opts.CREnabled {
+		return
+	}
+
+	// Set up a timeout on the whole operation.
+	ctx, cancel := context.WithTimeout(context.Background(), panicUploadMaxWait)
+	defer cancel()
+
+	// Print a notice if the upload takes a long time.
+	go func() {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(panicUploadNoticeWait):
+			l.Warnln("Uploading crash reports is taking a while, please wait...")
+		}
+	}()
+
+	// Report the panics.
+	dir := locations.GetBaseDir(locations.ConfigBaseDir)
+	uploadPanicLogs(ctx, opts.CRURL, dir)
+}

cmd/syncthing/monitor_test.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -14,6 +15,126 @@ import (
 	"time"
 )
 
+func TestRotatedFile(t *testing.T) {
+	// Verify that log rotation happens.
+
+	dir, err := ioutil.TempDir("", "syncthing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	open := func(name string) (io.WriteCloser, error) {
+		return os.Create(name)
+	}
+
+	logName := filepath.Join(dir, "log.txt")
+	testData := []byte("12345678\n")
+	maxSize := int64(len(testData) + len(testData)/2)
+
+	// We allow the log file plus two rotated copies.
+	rf, err := newRotatedFile(logName, open, maxSize, 2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Write some bytes.
+	if _, err := rf.Write(testData); err != nil {
+		t.Fatal(err)
+	}
+	// They should be in the log.
+	checkSize(t, logName, len(testData))
+	checkNotExist(t, logName+".0")
+
+	// Write some more bytes. We should rotate and write into a new file as the
+	// new bytes don't fit.
+	if _, err := rf.Write(testData); err != nil {
+		t.Fatal(err)
+	}
+	checkSize(t, logName, len(testData))
+	checkSize(t, numberedFile(logName, 0), len(testData))
+	checkNotExist(t, logName+".1")
+
+	// Write another byte. That should fit without causing an extra rotate.
+	_, _ = rf.Write([]byte{42})
+	checkSize(t, logName, len(testData)+1)
+	checkSize(t, numberedFile(logName, 0), len(testData))
+	checkNotExist(t, numberedFile(logName, 1))
+
+	// Write some more bytes. We should rotate and write into a new file as the
+	// new bytes don't fit.
+	if _, err := rf.Write(testData); err != nil {
+		t.Fatal(err)
+	}
+	checkSize(t, logName, len(testData))
+	checkSize(t, numberedFile(logName, 0), len(testData)+1) // the one we wrote extra to, now rotated
+	checkSize(t, numberedFile(logName, 1), len(testData))
+	checkNotExist(t, numberedFile(logName, 2))
+
+	// Write some more bytes. We should rotate and write into a new file as the
+	// new bytes don't fit.
+	if _, err := rf.Write(testData); err != nil {
+		t.Fatal(err)
+	}
+	checkSize(t, logName, len(testData))
+	checkSize(t, numberedFile(logName, 0), len(testData))
+	checkSize(t, numberedFile(logName, 1), len(testData)+1)
+	checkNotExist(t, numberedFile(logName, 2)) // exceeds maxFiles so deleted
+}
+
+func TestNumberedFile(t *testing.T) {
+	// Mostly just illustrates where the number ends up and makes sure it
+	// doesn't crash without an extension.
+
+	cases := []struct {
+		in  string
+		num int
+		out string
+	}{
+		{
+			in:  "syncthing.log",
+			num: 42,
+			out: "syncthing.42.log",
+		},
+		{
+			in:  filepath.Join("asdfasdf", "syncthing.log.txt"),
+			num: 42,
+			out: filepath.Join("asdfasdf", "syncthing.log.42.txt"),
+		},
+		{
+			in:  "syncthing-log",
+			num: 42,
+			out: "syncthing-log.42",
+		},
+	}
+
+	for _, tc := range cases {
+		res := numberedFile(tc.in, tc.num)
+		if res != tc.out {
+			t.Errorf("numberedFile(%q, %d) => %q, expected %q", tc.in, tc.num, res, tc.out)
+		}
+	}
+}
+
+func checkSize(t *testing.T, name string, size int) {
+	t.Helper()
+	info, err := os.Lstat(name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.Size() != int64(size) {
+		t.Errorf("%s wrong size: %d != expected %d", name, info.Size(), size)
+	}
+}
+
+func checkNotExist(t *testing.T, name string) {
+	t.Helper()
+	_, err := os.Lstat(name)
+	if !os.IsNotExist(err) {
+		t.Errorf("%s should not exist", name)
+	}
+}
+
 func TestAutoClosedFile(t *testing.T) {
 	os.RemoveAll("_autoclose")
 	defer os.RemoveAll("_autoclose")
@@ -22,7 +143,10 @@ func TestAutoClosedFile(t *testing.T) {
 	data := []byte("hello, world\n")
 
 	// An autoclosed file that closes very quickly
-	ac := newAutoclosedFile(file, time.Millisecond, time.Millisecond)
+	ac, err := newAutoclosedFile(file, time.Millisecond, time.Millisecond)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	// Write some data.
 	if _, err := ac.Write(data); err != nil {
@@ -64,21 +188,23 @@ func TestAutoClosedFile(t *testing.T) {
 	}
 
 	// Open the file again.
-	ac = newAutoclosedFile(file, time.Second, time.Second)
+	ac, err = newAutoclosedFile(file, time.Second, time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	// Write something
 	if _, err := ac.Write(data); err != nil {
 		t.Fatal(err)
 	}
 
-	// It should now contain only one write, because the first open
+	// It should now contain three writes, as the file is always opened for appending
-	// should be a truncate.
 	bs, err = ioutil.ReadFile(file)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(bs) != len(data) {
+	if len(bs) != 3*len(data) {
-		t.Fatalf("Write failed, expected %d bytes, not %d", len(data), len(bs))
+		t.Fatalf("Write failed, expected %d bytes, not %d", 3*len(data), len(bs))
 	}
 
 	// Close.

cmd/syncthing/perfstats_unix.go

@@ -38,7 +38,10 @@ func savePerfStats(file string) {
 
 	t0 := time.Now()
 	for t := range time.NewTicker(250 * time.Millisecond).C {
-		syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
+		if err := syscall.Getrusage(syscall.RUSAGE_SELF, &rusage); err != nil {
+			continue
+		}
+
 		curTime := time.Now().UnixNano()
 		timeDiff := curTime - prevTime
 		curUsage := rusage.Utime.Nano() + rusage.Stime.Nano()

cmd/syncthing/summaryservice.go

@@ -1,231 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"time"
-
-	"github.com/syncthing/syncthing/lib/events"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/sync"
-	"github.com/thejerf/suture"
-)
-
-// The folderSummaryService adds summary information events (FolderSummary and
-// FolderCompletion) into the event stream at certain intervals.
-type folderSummaryService struct {
-	*suture.Supervisor
-
-	cfg       configIntf
-	model     modelIntf
-	stop      chan struct{}
-	immediate chan string
-
-	// For keeping track of folders to recalculate for
-	foldersMut sync.Mutex
-	folders    map[string]struct{}
-
-	// For keeping track of when the last event request on the API was
-	lastEventReq    time.Time
-	lastEventReqMut sync.Mutex
-}
-
-func newFolderSummaryService(cfg configIntf, m modelIntf) *folderSummaryService {
-	service := &folderSummaryService{
-		Supervisor: suture.New("folderSummaryService", suture.Spec{
-			PassThroughPanics: true,
-		}),
-		cfg:             cfg,
-		model:           m,
-		stop:            make(chan struct{}),
-		immediate:       make(chan string),
-		folders:         make(map[string]struct{}),
-		foldersMut:      sync.NewMutex(),
-		lastEventReqMut: sync.NewMutex(),
-	}
-
-	service.Add(serviceFunc(service.listenForUpdates))
-	service.Add(serviceFunc(service.calculateSummaries))
-
-	return service
-}
-
-func (c *folderSummaryService) Stop() {
-	c.Supervisor.Stop()
-	close(c.stop)
-}
-
-// listenForUpdates subscribes to the event bus and makes note of folders that
-// need their data recalculated.
-func (c *folderSummaryService) listenForUpdates() {
-	sub := events.Default.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged | events.RemoteDownloadProgress | events.DeviceConnected | events.FolderWatchStateChanged)
-	defer events.Default.Unsubscribe(sub)
-
-	for {
-		// This loop needs to be fast so we don't miss too many events.
-
-		select {
-		case ev := <-sub.C():
-			if ev.Type == events.DeviceConnected {
-				// When a device connects we schedule a refresh of all
-				// folders shared with that device.
-
-				data := ev.Data.(map[string]string)
-				deviceID, _ := protocol.DeviceIDFromString(data["id"])
-
-				c.foldersMut.Lock()
-			nextFolder:
-				for _, folder := range c.cfg.Folders() {
-					for _, dev := range folder.Devices {
-						if dev.DeviceID == deviceID {
-							c.folders[folder.ID] = struct{}{}
-							continue nextFolder
-						}
-					}
-				}
-				c.foldersMut.Unlock()
-
-				continue
-			}
-
-			// The other events all have a "folder" attribute that they
-			// affect. Whenever the local or remote index is updated for a
-			// given folder we make a note of it.
-
-			data := ev.Data.(map[string]interface{})
-			folder := data["folder"].(string)
-
-			switch ev.Type {
-			case events.StateChanged:
-				if data["to"].(string) == "idle" && data["from"].(string) == "syncing" {
-					// The folder changed to idle from syncing. We should do an
-					// immediate refresh to update the GUI. The send to
-					// c.immediate must be nonblocking so that we can continue
-					// handling events.
-
-					c.foldersMut.Lock()
-					select {
-					case c.immediate <- folder:
-						delete(c.folders, folder)
-					default:
-						c.folders[folder] = struct{}{}
-					}
-					c.foldersMut.Unlock()
-				}
-
-			default:
-				// This folder needs to be refreshed whenever we do the next
-				// refresh.
-
-				c.foldersMut.Lock()
-				c.folders[folder] = struct{}{}
-				c.foldersMut.Unlock()
-			}
-
-		case <-c.stop:
-			return
-		}
-	}
-}
-
-// calculateSummaries periodically recalculates folder summaries and
-// completion percentage, and sends the results on the event bus.
-func (c *folderSummaryService) calculateSummaries() {
-	const pumpInterval = 2 * time.Second
-	pump := time.NewTimer(pumpInterval)
-
-	for {
-		select {
-		case <-pump.C:
-			t0 := time.Now()
-			for _, folder := range c.foldersToHandle() {
-				c.sendSummary(folder)
-			}
-
-			// We don't want to spend all our time calculating summaries. Lets
-			// set an arbitrary limit at not spending more than about 30% of
-			// our time here...
-			wait := 2*time.Since(t0) + pumpInterval
-			pump.Reset(wait)
-
-		case folder := <-c.immediate:
-			c.sendSummary(folder)
-
-		case <-c.stop:
-			return
-		}
-	}
-}
-
-// foldersToHandle returns the list of folders needing a summary update, and
-// clears the list.
-func (c *folderSummaryService) foldersToHandle() []string {
-	// We only recalculate summaries if someone is listening to events
-	// (a request to /rest/events has been made within the last
-	// pingEventInterval).
-
-	c.lastEventReqMut.Lock()
-	last := c.lastEventReq
-	c.lastEventReqMut.Unlock()
-	if time.Since(last) > defaultEventTimeout {
-		return nil
-	}
-
-	c.foldersMut.Lock()
-	res := make([]string, 0, len(c.folders))
-	for folder := range c.folders {
-		res = append(res, folder)
-		delete(c.folders, folder)
-	}
-	c.foldersMut.Unlock()
-	return res
-}
-
-// sendSummary send the summary events for a single folder
-func (c *folderSummaryService) sendSummary(folder string) {
-	// The folder summary contains how many bytes, files etc
-	// are in the folder and how in sync we are.
-	data, err := folderSummary(c.cfg, c.model, folder)
-	if err != nil {
-		return
-	}
-	events.Default.Log(events.FolderSummary, map[string]interface{}{
-		"folder":  folder,
-		"summary": data,
-	})
-
-	for _, devCfg := range c.cfg.Folders()[folder].Devices {
-		if devCfg.DeviceID.Equals(myID) {
-			// We already know about ourselves.
-			continue
-		}
-		if _, ok := c.model.Connection(devCfg.DeviceID); !ok {
-			// We're not interested in disconnected devices.
-			continue
-		}
-
-		// Get completion percentage of this folder for the
-		// remote device.
-		comp := jsonCompletion(c.model.Completion(devCfg.DeviceID, folder))
-		comp["folder"] = folder
-		comp["device"] = devCfg.DeviceID.String()
-		events.Default.Log(events.FolderCompletion, comp)
-	}
-}
-
-func (c *folderSummaryService) gotEventRequest() {
-	c.lastEventReqMut.Lock()
-	c.lastEventReq = time.Now()
-	c.lastEventReqMut.Unlock()
-}
-
-// serviceFunc wraps a function to create a suture.Service without stop
-// functionality.
-type serviceFunc func()
-
-func (f serviceFunc) Serve() { f() }
-func (f serviceFunc) Stop()  {}

cmd/todos/main.go

@@ -1,39 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build ignore
-
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"os"
-)
-
-func main() {
-	buf := make([]byte, 4096)
-	var err error
-	for err == nil {
-		n, err := io.ReadFull(os.Stdin, buf)
-		if n > 0 {
-			buf = buf[:n]
-			repl := bytes.Replace(buf, []byte("\n"), []byte("\r\n"), -1)
-			_, err = os.Stdout.Write(repl)
-			if err != nil {
-				fmt.Println(err)
-				os.Exit(1)
-			}
-		}
-		if err == io.EOF {
-			return
-		}
-		buf = buf[:cap(buf)]
-	}
-	fmt.Println(err)
-	os.Exit(1)
-}

cmd/uraggregate/main.go

@@ -181,7 +181,7 @@ func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) {
 		WHERE
 			DATE_TRUNC('day', Received) > $1
 			AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
-			AND Version like 'v0.%'
+			AND Version like 'v_.%'
 		GROUP BY Day, Ver
 		);
 	`, since)
@@ -199,7 +199,7 @@ func aggregateUserMovement(db *sql.DB) (int64, error) {
 		FROM Reports
 		WHERE
 			DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
-			AND Version like 'v0.%'
+			AND Version like 'v_.%'
 		ORDER BY Day
 	`)
 	if err != nil {
@@ -285,7 +285,7 @@ func aggregatePerformance(db *sql.DB, since time.Time) (int64, error) {
 		WHERE
 			DATE_TRUNC('day', Received) > $1
 			AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
-			AND Version like 'v0.%'
+			AND Version like 'v_.%'
 		GROUP BY Day
 		);
 	`, since)
@@ -315,7 +315,7 @@ func aggregateBlockStats(db *sql.DB, since time.Time) (int64, error) {
 			DATE_TRUNC('day', Received) > $1
 			AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
 			AND ReportVersion = 3
-			AND Version LIKE 'v0.%'
+			AND Version like 'v_.%'
 			AND Version NOT LIKE 'v0.14.40%'
 			AND Version NOT LIKE 'v0.14.39%'
 			AND Version NOT LIKE 'v0.14.38%'