cmd/syncthing: Rename --conf back to --config

This was inadvertently changed in the flag migration.

.deepsource.toml

@@ -0,0 +1,12 @@
+version = 1
+
+exclude_patterns = ["*.pb.go"]
+test_patterns = ["*_test.go"]
+
+[[analyzers]]
+name = "go"
+enabled = true
+
+  [analyzers.meta]
+  import_paths = ["github.com/syncthing/syncthing"]
+  build_tags = ["noassets"]

.github/FUNDING.yml

@@ -0,0 +1,11 @@
+github: syncthing
+custom: "https://syncthing.net/donations/"
+
+# patreon: # Replace with a single Patreon username
+# open_collective: # Replace with a single Open Collective username
+# ko_fi: # Replace with a single Ko-fi username
+# tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+# community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+# liberapay: # Replace with a single Liberapay username
+# issuehunt: # Replace with a single IssueHunt username
+# otechie: # Replace with a single Otechie username

.github/ISSUE_TEMPLATE.md

@@ -1,42 +0,0 @@
-### DO NOT REPORT SECURITY ISSUES IN THIS ISSUE TRACKER
-
-Instead, contact security@syncthing.net directly - see
-https://syncthing.net/security.html for more information.
-
-### DO NOT POST SUPPORT REQUESTS OR GENERAL QUESTIONS IN THIS ISSUE TRACKER
-
-Please use the forum at https://forum.syncthing.net/ where a large number of
-helpful people hang out. This issue tracker is for reporting bugs or feature
-requests directly to the developers. Worst case you might get a short
-"that's a bug, please report it on GitHub" response on the forum, in which
-case we thank you for your patience and following our advice. :)
-
-### Please use the correct issue tracker
-
-If your problem relates to a Syncthing wrapper or [sub-project](https://github.com/syncthing) such as [Syncthing for Android](https://github.com/syncthing/syncthing-android/issues), [SyncTrayzor](https://github.com/canton7/synctrayzor) or the [documentation](https://github.com/syncthing/docs/issues), please use their respective issue trackers.
-
-### Does your log mention database corruption?
-
-If your Syncthing log reports panics because of database corruption it is most likely a fault with your system's storage or memory. Affected log entries will contain lines starting with `panic: leveldb`. You will need to delete the index database to clear this, by running `syncthing -reset-database`.
-
-### Please do post actual bug reports and feature requests.
-
-If your issue is a bug report, replace this boilerplate with a description
-of the problem, being sure to include at least:
-
- - what happened,
- - what you expected to happen instead, and
- - any steps to reproduce the problem.
-
-Also fill out the version information below and add log output or
-screenshots as appropriate.
-
-If your issue is a feature request, simply replace this template text in
-its entirety.
-
-### Version Information
-
-Syncthing Version: v0.x.y
-OS Version: Windows 7 / Ubuntu 14.04 / ...
-Browser Version: (if applicable, for GUI issues)
-

.github/ISSUE_TEMPLATE/01-feature.md

@@ -0,0 +1,13 @@
+---
+name: Feature request
+about: If you're just not sure how to do something, see "ask a question".
+labels: enhancement, needs-triage
+---
+
+### Include required information
+
+Please be sure to include at least:
+
+ - what problem your new feature would solve
+ - how or why you think it is generally useful (i.e., not just for you)
+ - what alternatives or workarounds you considered

.github/ISSUE_TEMPLATE/02-bug.md

@@ -0,0 +1,23 @@
+---
+name: Bug report
+about: If you're actually looking for support, see "ask a question".
+labels: bug, needs-triage
+---
+
+### Does your log mention database corruption?
+
+If your Syncthing log reports panics because of database corruption it is
+most likely a fault with your system's storage or memory. Affected log
+entries will contain lines starting with `panic: leveldb`. You will need to
+delete the index database to clear this, by running `syncthing
+-reset-database`.
+
+### Include required information
+
+Please be sure to include at least:
+
+ - which version of Syncthing and what operating system you are using
+ - browser and version, if applicable
+ - what happened,
+ - what you expected to happen instead, and
+ - any steps to reproduce the problem.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: I need help / I have a question
+    url: https://forum.syncthing.net/
+    about: Ask questions, get support, and discuss with other community members.
+  - name: Android issues
+    url: https://github.com/syncthing/syncthing-android/issues/
+    about: The Android app has its own issue tracker.

.github/SECURITY.md

@@ -0,0 +1,10 @@
+## Reporting a Vulnerability
+
+If you believe that you've found a Syncthing-related security vulnerability,
+please report it by sending email to the address security@syncthing.net. The
+[PGP key for security@syncthing.net
+(B683AD7B76CAB013)](https://syncthing.net/security-key.txt) can be used to
+send encrypted mail or to verify responses received from that address.
+
+You can read more about Syncthing security at
+https://syncthing.net/security/.

.gitignore

@@ -15,12 +15,7 @@ coverage.xml
 syncthing.sig
 RELEASE
 deb
-lib/auto/gui.files.go
-snapcraft.yaml
-prime/
-snap/
-parts/
-stage/
-*.snap
 *.bz2
 /repos
+/proto/scripts/protoc-gen-gosyncthing
+/gui/next-gen-gui

.golangci.yml

@@ -1,5 +1,26 @@
+linters-settings:
+  maligned:
+    suggest-new: true
+
+linters:
+  enable-all: true
+  disable:
+    - goimports
+    - depguard
+    - lll
+    - gochecknoinits
+    - gochecknoglobals
+    - gofmt
+    - scopelint
+    - gocyclo
+    - funlen
+    - wsl
+    - gocognit
+    - godox
+
 service:
-  golangci-lint-version: 1.13.x
+  golangci-lint-version: 1.21.x
   prepare:
+    - rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
+    - GO111MODULE=on go mod vendor
     - go run build.go assets
-    - GO111MODULE=on go mod vendor

AUTHORS

@@ -16,20 +16,30 @@
 Aaron Bieber (qbit) <qbit@deftly.net>
 Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com> <adam@proactiveservices.co.uk>
 Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
+Alan Pope <alan@popey.com>
+Alberto Donato <albertodonato@users.noreply.github.com>
 Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
+Alex Lindeman <139387+aelindeman@users.noreply.github.com>
+Alex Xu <alex.hello71@gmail.com>
 Alexander Graf (alex2108) <register-github@alex-graf.de>
 Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
+Aman Gupta <aman@tmm1.net>
 Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
 andresvia <andres.via@gmail.com>
 Andrew Dunham (andrew-d) <andrew@du.nham.ca>
 Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
 Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
+André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
 andyleap <andyleap@gmail.com>
+Anjan Momi <anjan@momi.ca>
 Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
 Antony Male (canton7) <antony.male@gmail.com>
 Aranjedeath <Aranjedeath@users.noreply.github.com>
+Arkadiusz Tymiński <gevleeog@gmail.com>
 Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
-Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com>
+Artur Zubilewicz <AkaZecik@users.noreply.github.com>
+Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
+Aurélien Rainone <476650+arl@users.noreply.github.com>
 BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
 Bart De Vries (mogwa1) <devriesb@gmail.com>
 Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
@@ -40,21 +50,29 @@ Benedikt Heine (bebehei) <bebe@bebehei.de>
 Benedikt Morbach <benedikt.morbach@googlemail.com>
 Benno Fünfstück <benno.fuenfstueck@gmail.com>
 Benny Ng (tpng) <benny.tpng@gmail.com>
+boomsquared <54829195+boomsquared@users.noreply.github.com>
+Boqin Qin <bobbqqin@bupt.edu.cn>
 Boris Rybalkin <ribalkin@gmail.com>
 Brandon Philips (philips) <brandon@ifup.org>
 Brendan Long (brendanlong) <self@brendanlong.com>
 Brian R. Becker (brbecker) <brbecker@gmail.com>
+bt90 <btom1990@googlemail.com>
 Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
-Carsten Hagemann (Moter8) <moter8@gmail.com>
+Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
 Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
 Cedric Staniewski (xduugu) <cedric@gmx.ca>
+chenrui <rui@meetup.com>
+Choongkyu <choongkyu.kim+gh@gmail.com> <vapidlyrapid+gh@gmail.com>
 Chris Howie (cdhowie) <me@chrishowie.com>
 Chris Joel (cdata) <chris@scriptolo.gy>
 Chris Tonkinson <chris@masterbran.ch>
+Christian Prescott <me@christianprescott.com>
 chucic <chucic@seznam.cz>
 Colin Kennedy (moshen) <moshen.colin@gmail.com>
-Cromefire_ <tim.l@nghorst.net>
+Cromefire_ <tim.l@nghorst.net> <26320625+cromefire@users.noreply.github.com>
+Cyprien Devillez <cypx@users.noreply.github.com>
 Dale Visser <dale.visser@live.com>
+Dan <benda.daniel@gmail.com>
 Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
 Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
 Daniel Martí (mvdan) <mvdan@mvdan.cc>
@@ -62,54 +80,73 @@ Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
 David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
 Denis A. (dva) <denisva@gmail.com>
 Dennis Wilson (snnd) <dw@risu.io>
+dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com> <27856297+dependabot-preview[bot]@users.noreply.github.com>
+dependabot[bot] <dependabot[bot]@users.noreply.github.com>
 derekriemer <derek.riemer@colorado.edu>
 desbma <desbma@users.noreply.github.com>
 Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
+Domenic Horner <domenic@tgxn.net>
 Dominik Heidler (asdil12) <dominik@heidler.eu>
 Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
 Elliot Huffman <thelich2@gmail.com>
 Emil Hessman (ceh) <emil@hessman.se>
+Eric Lesiuta <elesiuta@gmail.com>
 Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
 Evgeny Kuznetsov <evgeny@kuznetsov.md>
 Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
 Felix Ableitner (Nutomic) <me@nutomic.com>
+Felix Lampe <mail@flampe.de>
 Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
 Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
 Frank Isemann (fti7) <frank@isemann.name>
 georgespatton <georgespatton@users.noreply.github.com>
+ghjklw <malo@jaffre.info>
 Gilli Sigurdsson (gillisig) <gilli@vx.is>
+Gleb Sinyavskiy <zhulik.gleb@gmail.com>
 Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
+greatroar <61184462+greatroar@users.noreply.github.com>
 Han Boetes <han@boetes.org>
+HansK-p <42314815+HansK-p@users.noreply.github.com>
 Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
 Heiko Zuerker (Smiley73) <heiko@zuerker.org>
 Hugo Locurcio <hugo.locurcio@hugo.pro>
 Iain Barnett <iainspeed@gmail.com>
 Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
+Ilya Brin <464157+ilyabrin@users.noreply.github.com>
 Iskander Sharipov (Alex) <quasilyte@gmail.com>
 Jaakko Hannikainen (jgke) <jgke@jgke.fi>
 Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
+Jack Croft <jccroft1@users.noreply.github.com>
+Jacob <jyundt@gmail.com>
 Jake Peterson (acogdev) <jake@acogdev.com>
 Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
 James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
 janost <janost@tuta.io>
+Jaroslav Lichtblau <svetlemodry@users.noreply.github.com>
 Jaroslav Malec (dzarda) <dzardacz@gmail.com>
 jaseg <githubaccount@jaseg.net>
 Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
+jelle van der Waa <jelle@vdwaa.nl>
 Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
 Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
+Jesse Lucas <jesse@jesselucas.com>
 Jochen Voss (seehuhn) <voss@seehuhn.de>
 Johan Andersson <j@i19.se>
 Johan Vromans (sciurius) <jvromans@squirrel.nl>
 John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
 Jonas Thelemann <e-mail@jonas-thelemann.de>
+Jonathan <artback@protonmail.com>
 Jonathan Cross <jcross@gmail.com>
 Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
 Jörg Thalheim <Mic92@users.noreply.github.com>
+Jędrzej Kula <kula.jedrek@gmail.com>
+Kalle Laine <pahakalle@protonmail.com>
 Karol Różycki (krozycki) <rozycki.karol@gmail.com>
 Keith Turner <kturner@apache.org>
 Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
 Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
 Kevin Allen (ironmig) <kma1660@gmail.com>
+Kevin Bushiri (keevBush) <keevbush@gmail.com> <36192217+keevBush@users.noreply.github.com>
 Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
 klemens <ka7@github.com>
 Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
@@ -120,13 +157,17 @@ Leo Arias (elopio) <yo@elopio.net>
 Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
 Lode Hoste (Zillode) <zillode@zillode.be>
 Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
+Lukas Lihotzki <lukas@lihotzki.de>
 Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
 Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
 Marc Pujol (kilburn) <kilburn@la3.org>
 Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
 marco-m <marco.molteni@laposte.net>
+Marcus Legendre <marcus.legendre@gmail.com>
+Mario Majila <mariustshipichik@gmail.com>
 Mark Pulford (mpx) <mark@kyne.com.au>
 Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
+Mateusz Ż <thedead4fun@live.com>
 Matic Potočnik <hairyfotr@gmail.com>
 Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
 Matt Robenolt <matt@ydekproductions.com>
@@ -137,19 +178,30 @@ MaximAL <almaximal@ya.ru>
 Maxime Thirouin <m@moox.io>
 Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
 Michael Ploujnikov (plouj) <ploujj@gmail.com>
+Michael Rienstra <mrienstra@gmail.com>
 Michael Tilli (pyfisch) <pyfisch@gmail.com>
 Mike Boone <mike@boonedocks.net>
 MikeLund <MikeLund@users.noreply.github.com>
+MikolajTwarog <43782609+MikolajTwarog@users.noreply.github.com>
+Mingxuan Lin <gdlmx@users.noreply.github.com>
+mv1005 <49659413+mv1005@users.noreply.github.com>
 Nate Morrison (nrm21) <natemorrison@gmail.com>
 Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
 Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
 Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
+Nicolas Perraut <n.perraut@gmail.com>
 Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
 Nils Jakobi (thunderstorm99) <jakobi.nils@gmail.com>
+NinoM4ster <ninom4ster@gmail.com>
+Nitroretro <43112364+Nitroretro@users.noreply.github.com>
 NoLooseEnds <jon.koslung@gmail.com>
+Oliver Freyermuth <o.freyermuth@googlemail.com>
 otbutz <tbutz@optitool.de>
+Otiel <Otiel@users.noreply.github.com>
 Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
+Pablo <pbaeyens31+github@gmail.com>
 Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
+Paul Brit <paulbrit44@gmail.com>
 Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
 Paweł Rozlach <vespian@users.noreply.github.com>
 perewa <cavalcante.ten@gmail.com>
@@ -157,36 +209,48 @@ Peter Badida <KeyWeeUsr@users.noreply.github.com>
 Peter Dave Hello <hsu@peterdavehello.org>
 Peter Hoeg (peterhoeg) <peter@speartail.com>
 Peter Marquardt (wwwutz) <wwwutz@gmail.com> <wwwutz@googlemail.com>
+Phani Rithvij <phanirithvij2000@gmail.com>
 Phil Davis <phil.davis@inf.org>
 Philippe Schommers (filoozoom) <philippe@schommers.be>
 Phill Luby (pluby) <phill.luby@newredo.com>
 Pier Paolo Ramon <ramonpierre@gmail.com>
 Piotr Bejda (piobpl) <piotrb10@gmail.com>
 Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
+Quentin Hibon <qh.public@yahoo.com>
+Rahmi Pruitt <rjpruitt16@gmail.com>
 Richard Hartmann <RichiH@users.noreply.github.com>
 Robert Carosi (nov1n) <robert@carosi.nl>
+Roberto Santalla <roobre@users.noreply.github.com>
+Robin Schoonover <robin@cornhooves.org>
 Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
 Ross Smith II (rasa) <ross@smithii.com>
 rubenbe <github-com-00ff86@vandamme.email>
+Ruslan Yevdokymov <38809160+ruslanye@users.noreply.github.com>
 Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
 Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
 Scott Klupfel (kluppy) <kluppy@going2blue.com>
 Sergey Mishin (ralder) <ralder@yandex.ru>
+Shaarad Dalvi <60266155+shaaraddalvi@users.noreply.github.com>
 Simon Frei (imsodin) <freisim93@gmail.com>
+Simon Mwepu <simonmwepu@gmail.com>
 Sly_tom_cat <slytomcat@mail.ru>
 Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
 Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
+Steven Eckhoff <steven.eckhoff.opensource@gmail.com>
 Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
 Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
 Thomas Hipp <thomashipp@gmail.com>
 Tim Abell (timabell) <tim@timwise.co.uk>
 Tim Howes (timhowes) <timhowes@berkeley.edu>
+Tobias Klauser <tobias.klauser@gmail.com>
 Tobias Nygren (tnn2) <tnn@nygren.pp.se>
 Tobias Tom (tobiastom) <t.tom@succont.de>
-Tomas Cerveny (kozec) <kozec@kozec.com>
+Tom Jakubowski <tom@crystae.net>
+Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com> <twilczynski@naver.com>
 Tommy Thorn <tommy-github-email@thorn.ws>
 Tully Robinson (tojrobinson) <tully@tojr.org>
 Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
+Tyler Kropp <kropptyler@gmail.com>
 Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
 Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
 Victor Buinsky (buinsky) <vix_booja@tut.by>
@@ -194,7 +258,9 @@ Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
 Vladimir Rusinov <vrusinov@google.com>
 wangguoliang <liangcszzu@163.com>
 William A. Kennington III (wkennington) <william@wkennington.com>
+wouter bolsterlee <wouter@bolsterl.ee>
 Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
+xarx00 <xarx00@users.noreply.github.com>
 Xavier O. (damajor) <damajor@gmail.com>
 xjtdy888 (xjtdy888) <xjtdy888@163.com>
 Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>

Dockerfile

@@ -1,4 +1,5 @@
-FROM golang:1.12 AS builder
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
 
 WORKDIR /src
 COPY . .
@@ -10,23 +11,19 @@ RUN rm -f syncthing && go run build.go -no-upgrade build syncthing
 
 FROM alpine
 
-EXPOSE 8384 22000 21027/udp
+EXPOSE 8384 22000/tcp 22000/udp 21027/udp
 
 VOLUME ["/var/syncthing"]
 
-RUN apk add --no-cache ca-certificates su-exec
+RUN apk add --no-cache ca-certificates su-exec tzdata
 
 COPY --from=builder /src/syncthing /bin/syncthing
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
 
-ENV PUID=1000 PGID=1000
+ENV PUID=1000 PGID=1000 HOME=/var/syncthing
 
 HEALTHCHECK --interval=1m --timeout=10s \
-  CMD nc -z localhost 8384 || exit 1
+  CMD nc -z 127.0.0.1 8384 || exit 1
 
-ENTRYPOINT \
+ENV STGUIADDRESS=0.0.0.0:8384
-  chown "${PUID}:${PGID}" /var/syncthing \
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]
-  && su-exec "${PUID}:${PGID}" \
-     env HOME=/var/syncthing \
-     /bin/syncthing \
-       -home /var/syncthing/config \
-       -gui-address 0.0.0.0:8384

Dockerfile.builder

@@ -0,0 +1,9 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION
+
+# FPM to build Debian packages
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	locales rubygems ruby-dev build-essential git \
+	&& apt-get clean \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& gem install --no-ri --no-rdoc fpm

Dockerfile.buildx

@@ -0,0 +1,19 @@
+FROM alpine
+ARG TARGETARCH
+
+EXPOSE 8384 22000/tcp 22000/udp 21027/udp
+
+VOLUME ["/var/syncthing"]
+
+RUN apk add --no-cache ca-certificates su-exec tzdata
+
+COPY ./syncthing-linux-$TARGETARCH /bin/syncthing
+COPY ./script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/syncthing
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z 127.0.0.1 8384 || exit 1
+
+ENV STGUIADDRESS=0.0.0.0:8384
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]

Dockerfile.stdiscosrv

@@ -0,0 +1,29 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f stdiscosrv && go run build.go -no-upgrade build stdiscosrv
+
+FROM alpine
+
+EXPOSE 19200 8443
+
+VOLUME ["/var/stdiscosrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/stdiscosrv /bin/stdiscosrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/stdiscosrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 8443 || exit 1
+
+WORKDIR /var/stdiscosrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/stdiscosrv"]

Dockerfile.strelaysrv

@@ -0,0 +1,29 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f strelaysrv && go run build.go -no-upgrade build strelaysrv
+
+FROM alpine
+
+EXPOSE 22067 22070
+
+VOLUME ["/var/strelaysrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/strelaysrv /bin/strelaysrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/strelaysrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 22067 || exit 1
+
+WORKDIR /var/strelaysrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaysrv"]

README-Docker.md

@@ -13,12 +13,16 @@ altered with the ``PUID`` and ``PGID`` environment variables.
 
 ```
 $ docker pull syncthing/syncthing
-$ docker run -p 8384:8384 -p 22000:22000 \
+$ docker run -p 8384:8384 -p 22000:22000/tcp -p 22000:22000/udp \
     -v /wherever/st-sync:/var/syncthing \
     syncthing/syncthing:latest
 ```
 
-Note that local device discovery will not work with the above command, resulting in poor local transfer rates if local device addresses are not manually configured.
+## Discovery
+
+Note that local device discovery will not work with the above command,
+resulting in poor local transfer rates if local device addresses are not
+manually configured.
 
 To allow local discovery, the docker host network can be used instead:
 
@@ -32,3 +36,24 @@ $ docker run --network=host \
 Be aware that syncthing alone is now in control of what interfaces and ports it
 listens on. You can edit the syncthing configuration to change the defaults if
 there are conflicts.
+
+## GUI Security
+
+By default Syncthing inside the Docker image listens on 0.0.0.0:8384 to
+allow GUI connections via the Docker proxy. This is set by the
+`STGUIADDRESS` environment variable in the Dockerfile, as it differs from
+what Syncthing would otherwise use by default. This means you should set up
+authentication in the GUI, like for any other externally reachable Syncthing
+instance. If you do not require the GUI, or you use host networking, you can
+unset the `STGUIADDRESS` variable to have Syncthing fall back to listening
+on 127.0.0.1:
+
+```
+$ docker pull syncthing/syncthing
+$ docker run -e STGUIADDRESS= \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+With the environment variable unset Syncthing will follow what is set in the
+configuration file / GUI settings dialog.

README.md

@@ -2,7 +2,6 @@
 
 ---
 
-[![Latest Downloads](https://img.shields.io/badge/latest-downloads-brightgreen.svg?style=flat-square)](https://build.syncthing.net/latest/)
 [![Latest Linux & Cross Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildLinuxCross.svg?style=flat-square&label=linux+%26+cross+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildLinuxCross&guest=1)
 [![Latest Windows Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildWindows.svg?style=flat-square&label=windows+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildWindows&guest=1)
 [![Latest Mac Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildMac.svg?style=flat-square&label=mac+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildMac&guest=1)
@@ -63,6 +62,10 @@ There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory][3]. There are also several [GUI
 implementations][11] for Windows, Mac and Linux.
 
+## Docker
+
+To run Syncthing in Docker, see [the Docker README][16].
+
 ## Vote on features/bugs
 
 We'd like to encourage you to [vote][12] on issues that matter to you.
@@ -98,17 +101,18 @@ All code is licensed under the [MPLv2 License][7].
 
 [1]: https://docs.syncthing.net/specs/bep-v1.html
 [2]: https://docs.syncthing.net/intro/getting-started.html
-[3]: https://github.com/syncthing/syncthing/blob/master/etc
+[3]: https://github.com/syncthing/syncthing/blob/main/etc
 [4]: https://www.freenode.net/
 [5]: https://docs.syncthing.net/dev/building.html
 [6]: https://docs.syncthing.net/
-[7]: https://github.com/syncthing/syncthing/blob/master/LICENSE
+[7]: https://github.com/syncthing/syncthing/blob/main/LICENSE
 [8]: https://forum.syncthing.net/
 [9]: https://kiwiirc.com/client/irc.freenode.net/#syncthing
 [10]: https://github.com/syncthing/syncthing/issues
 [11]: https://docs.syncthing.net/users/contrib.html#gui-wrappers
 [12]: https://www.bountysource.com/teams/syncthing/issues
-[13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
+[13]: https://github.com/syncthing/syncthing/blob/main/GOALS.md
 [14]: assets/logo-text-128.png
 [15]: https://syncthing.net/
+[16]: https://github.com/syncthing/syncthing/blob/main/README-Docker.md
 

build.go

@@ -15,6 +15,7 @@ import (
 	"compress/flate"
 	"compress/gzip"
 	"crypto/sha256"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -29,39 +30,41 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"text/template"
 	"time"
 )
 
 var (
-	versionRe        = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
+	goarch         string
-	goarch           string
+	goos           string
-	goos             string
+	noupgrade      bool
-	noupgrade        bool
+	version        string
-	version          string
+	goCmd          string
-	goCmd            string
+	race           bool
-	goVersion        float64
+	debug          = os.Getenv("BUILDDEBUG") != ""
-	race             bool
+	extraTags      string
-	debug            = os.Getenv("BUILDDEBUG") != ""
+	installSuffix  string
-	extraTags        string
+	pkgdir         string
-	installSuffix    string
+	cc             string
-	pkgdir           string
+	run            string
-	cc               string
+	benchRun       string
-	debugBinary      bool
+	debugBinary    bool
-	coverage         bool
+	coverage       bool
-	timeout          = "120s"
+	timeout        = "120s"
-	gogoProtoVersion = "v1.2.0"
+	numVersions    = 5
+	withNextGenGUI = os.Getenv("BUILD_NEXT_GEN_GUI") != ""
 )
 
 type target struct {
 	name              string
 	debname           string
 	debdeps           []string
+	debpre            string
 	debpost           string
 	description       string
-	buildPkg          string
+	buildPkgs         []string
 	binaryName        string
 	archiveFiles      []archiveFile
+	systemdServices   []string
 	installationFiles []archiveFile
 	tags              []string
 }
@@ -75,9 +78,8 @@ type archiveFile struct {
 var targets = map[string]target{
 	"all": {
 		// Only valid for the "build" and "install" commands as it lacks all
-		// the archive creation stuff.
+		// the archive creation stuff. buildPkgs gets filled out in init()
-		buildPkg: "github.com/syncthing/syncthing/cmd/...",
+		tags: []string{"purego"},
-		tags:     []string{"purego"},
 	},
 	"syncthing": {
 		// The default target for "build", "install", "tar", "zip", "deb", etc.
@@ -86,7 +88,7 @@ var targets = map[string]target{
 		debdeps:     []string{"libc6", "procps"},
 		debpost:     "script/post-upgrade",
 		description: "Open Source Continuous File Synchronization",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/syncthing",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/syncthing"},
 		binaryName:  "syncthing", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -113,6 +115,7 @@ var targets = map[string]target{
 			{src: "etc/linux-systemd/system/syncthing@.service", dst: "deb/lib/systemd/system/syncthing@.service", perm: 0644},
 			{src: "etc/linux-systemd/system/syncthing-resume.service", dst: "deb/lib/systemd/system/syncthing-resume.service", perm: 0644},
 			{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0644},
+			{src: "etc/linux-sysctl/30-syncthing.conf", dst: "deb/usr/lib/sysctl.d/30-syncthing.conf", perm: 0644},
 			{src: "etc/firewall-ufw/syncthing", dst: "deb/etc/ufw/applications.d/syncthing", perm: 0644},
 			{src: "etc/linux-desktop/syncthing-start.desktop", dst: "deb/usr/share/applications/syncthing-start.desktop", perm: 0644},
 			{src: "etc/linux-desktop/syncthing-ui.desktop", dst: "deb/usr/share/applications/syncthing-ui.desktop", perm: 0644},
@@ -128,8 +131,9 @@ var targets = map[string]target{
 		name:        "stdiscosrv",
 		debname:     "syncthing-discosrv",
 		debdeps:     []string{"libc6"},
+		debpre:      "cmd/stdiscosrv/scripts/preinst",
 		description: "Syncthing Discovery Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/stdiscosrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/stdiscosrv"},
 		binaryName:  "stdiscosrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -137,12 +141,17 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
+		systemdServices: []string{
+			"cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service",
+		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
 			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0644},
 			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0644},
 			{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-discosrv", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv", dst: "deb/etc/ufw/applications.d/stdiscosrv", perm: 0644},
 		},
 		tags: []string{"purego"},
 	},
@@ -150,8 +159,9 @@ var targets = map[string]target{
 		name:        "strelaysrv",
 		debname:     "syncthing-relaysrv",
 		debdeps:     []string{"libc6"},
+		debpre:      "cmd/strelaysrv/scripts/preinst",
 		description: "Syncthing Relay Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/strelaysrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaysrv"},
 		binaryName:  "strelaysrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -160,6 +170,9 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
+		systemdServices: []string{
+			"cmd/strelaysrv/etc/linux-systemd/strelaysrv.service",
+		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
 			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0644},
@@ -167,6 +180,8 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0644},
 			{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0644},
+			{src: "cmd/strelaysrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-relaysrv", perm: 0644},
+			{src: "cmd/strelaysrv/etc/firewall-ufw/strelaysrv", dst: "deb/etc/ufw/applications.d/strelaysrv", perm: 0644},
 		},
 	},
 	"strelaypoolsrv": {
@@ -174,7 +189,7 @@ var targets = map[string]target{
 		debname:     "syncthing-relaypoolsrv",
 		debdeps:     []string{"libc6"},
 		description: "Syncthing Relay Pool Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/strelaypoolsrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaypoolsrv"},
 		binaryName:  "strelaypoolsrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -200,11 +215,25 @@ type dependencyRepo struct {
 }
 
 var dependencyRepos = []dependencyRepo{
-	{path: "protobuf", repo: "https://github.com/gogo/protobuf.git", commit: gogoProtoVersion},
 	{path: "xdr", repo: "https://github.com/calmh/xdr.git", commit: "08e072f9cb16"},
 }
 
-func init() {
+func initTargets() {
+	all := targets["all"]
+	pkgs, _ := filepath.Glob("cmd/*")
+	for _, pkg := range pkgs {
+		pkg = filepath.Base(pkg)
+		if strings.HasPrefix(pkg, ".") {
+			// ignore dotfiles
+			continue
+		}
+		if noupgrade && pkg == "stupgrades" {
+			continue
+		}
+		all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/cmd/%s", pkg))
+	}
+	targets["all"] = all
+
 	// The "syncthing" target includes a few more files found in the "etc"
 	// and "extra" dirs.
 	syncthingPkg := targets["syncthing"]
@@ -232,6 +261,8 @@ func main() {
 		}()
 	}
 
+	initTargets()
+
 	// Invoking build.go with no parameters at all builds everything (incrementally),
 	// which is what you want for maximum error checking during development.
 	if flag.NArg() == 0 {
@@ -254,29 +285,31 @@ func main() {
 }
 
 func runCommand(cmd string, target target) {
+	var tags []string
+	if noupgrade {
+		tags = []string{"noupgrade"}
+	}
+	tags = append(tags, strings.Fields(extraTags)...)
+
 	switch cmd {
 	case "install":
-		var tags []string
-		if noupgrade {
-			tags = []string{"noupgrade"}
-		}
-		tags = append(tags, strings.Fields(extraTags)...)
 		install(target, tags)
 		metalintShort()
 
 	case "build":
-		var tags []string
-		if noupgrade {
-			tags = []string{"noupgrade"}
-		}
-		tags = append(tags, strings.Fields(extraTags)...)
 		build(target, tags)
 
 	case "test":
-		test("github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
+		test(strings.Fields(extraTags), "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
 
 	case "bench":
-		bench("github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
+		bench(strings.Fields(extraTags), "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
+
+	case "integration":
+		integration(false)
+
+	case "integrationbench":
+		integration(true)
 
 	case "assets":
 		rebuildAssets()
@@ -284,6 +317,9 @@ func runCommand(cmd string, target target) {
 	case "proto":
 		proto()
 
+	case "testmocks":
+		testmocks()
+
 	case "translate":
 		translate()
 
@@ -291,17 +327,14 @@ func runCommand(cmd string, target target) {
 		transifex()
 
 	case "tar":
-		buildTar(target)
+		buildTar(target, tags)
 
 	case "zip":
-		buildZip(target)
+		buildZip(target, tags)
 
 	case "deb":
 		buildDeb(target)
 
-	case "snap":
-		buildSnap(target)
-
 	case "vet":
 		metalintShort()
 
@@ -314,6 +347,20 @@ func runCommand(cmd string, target target) {
 	case "version":
 		fmt.Println(getVersion())
 
+	case "changelog":
+		vers, err := currentAndLatestVersions(numVersions)
+		if err != nil {
+			log.Fatal(err)
+		}
+		for _, ver := range vers {
+			underline := strings.Repeat("=", len(ver))
+			msg, err := tagMessage(ver)
+			if err != nil {
+				log.Fatal(err)
+			}
+			fmt.Printf("%s\n%s\n\n%s\n\n", ver, underline, msg)
+		}
+
 	default:
 		log.Fatalf("Unknown command %q", cmd)
 	}
@@ -332,13 +379,18 @@ func parseFlags() {
 	flag.StringVar(&cc, "cc", os.Getenv("CC"), "Set CC environment variable for `go build`")
 	flag.BoolVar(&debugBinary, "debug-binary", debugBinary, "Create unoptimized binary to use with delve, set -gcflags='-N -l' and omit -ldflags")
 	flag.BoolVar(&coverage, "coverage", coverage, "Write coverage profile of tests to coverage.txt")
+	flag.IntVar(&numVersions, "num-versions", numVersions, "Number of versions for changelog command")
+	flag.StringVar(&run, "run", "", "Specify which tests to run")
+	flag.StringVar(&benchRun, "bench", "", "Specify which benchmarks to run")
+	flag.BoolVar(&withNextGenGUI, "with-next-gen-gui", withNextGenGUI, "Also build 'newgui'")
 	flag.Parse()
 }
 
-func test(pkgs ...string) {
+func test(tags []string, pkgs ...string) {
 	lazyRebuildAssets()
 
-	args := []string{"test", "-short", "-timeout", timeout, "-tags", "purego"}
+	tags = append(tags, "purego")
+	args := []string{"test", "-short", "-timeout", timeout, "-tags", strings.Join(tags, " ")}
 
 	if runtime.GOARCH == "amd64" {
 		switch runtime.GOOS {
@@ -348,18 +400,59 @@ func test(pkgs ...string) {
 	}
 
 	if coverage {
-		args = append(args, "-covermode", "atomic", "-coverprofile", "coverage.txt")
+		args = append(args, "-covermode", "atomic", "-coverprofile", "coverage.txt", "-coverpkg", strings.Join(pkgs, ","))
 	}
 
+	args = append(args, runArgs()...)
+
 	runPrint(goCmd, append(args, pkgs...)...)
 }
 
-func bench(pkgs ...string) {
+func bench(tags []string, pkgs ...string) {
 	lazyRebuildAssets()
-	runPrint(goCmd, append([]string{"test", "-run", "NONE", "-bench", "."}, pkgs...)...)
+	args := append([]string{"test", "-run", "NONE", "-tags", strings.Join(tags, " ")}, benchArgs()...)
+	runPrint(goCmd, append(args, pkgs...)...)
+}
+
+func integration(bench bool) {
+	lazyRebuildAssets()
+	args := []string{"test", "-v", "-timeout", "60m", "-tags"}
+	tags := "purego,integration"
+	if bench {
+		tags += ",benchmark"
+	}
+	args = append(args, tags)
+	args = append(args, runArgs()...)
+	if bench {
+		if run == "" {
+			args = append(args, "-run", "Benchmark")
+		}
+		args = append(args, benchArgs()...)
+	}
+	args = append(args, "./test")
+	fmt.Println(args)
+	runPrint(goCmd, args...)
+}
+
+func runArgs() []string {
+	if run == "" {
+		return nil
+	}
+	return []string{"-run", run}
+}
+
+func benchArgs() []string {
+	if benchRun == "" {
+		return []string{"-bench", "."}
+	}
+	return []string{"-bench", benchRun}
 }
 
 func install(target target, tags []string) {
+	if (target.name == "syncthing" || target.name == "") && !withNextGenGUI {
+		log.Println("Notice: Next generation GUI will not be built; see --with-next-gen-gui.")
+	}
+
 	lazyRebuildAssets()
 
 	tags = append(target.tags, tags...)
@@ -370,12 +463,7 @@ func install(target target, tags []string) {
 	}
 	os.Setenv("GOBIN", filepath.Join(cwd, "bin"))
 
-	args := []string{"install", "-v"}
+	setBuildEnvVars()
-	args = appendParameters(args, tags, target)
-
-	os.Setenv("GOOS", goos)
-	os.Setenv("GOARCH", goarch)
-	os.Setenv("CC", cc)
 
 	// On Windows generate a special file which the Go compiler will
 	// automatically use when generating Windows binaries to set things like
@@ -388,22 +476,22 @@ func install(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
+	args := []string{"install", "-v", "-trimpath"}
+	args = appendParameters(args, tags, target.buildPkgs...)
 	runPrint(goCmd, args...)
 }
 
 func build(target target, tags []string) {
-	lazyRebuildAssets()
+	if (target.name == "syncthing" || target.name == "") && !withNextGenGUI {
+		log.Println("Notice: Next generation GUI will not be built; see --with-next-gen-gui.")
+	}
 
+	lazyRebuildAssets()
 	tags = append(target.tags, tags...)
 
 	rmr(target.BinaryName())
 
-	args := []string{"build", "-v"}
+	setBuildEnvVars()
-	args = appendParameters(args, tags, target)
-
-	os.Setenv("GOOS", goos)
-	os.Setenv("GOARCH", goarch)
-	os.Setenv("CC", cc)
 
 	// On Windows generate a special file which the Go compiler will
 	// automatically use when generating Windows binaries to set things like
@@ -420,10 +508,25 @@ func build(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
+	args := []string{"build", "-v", "-trimpath"}
+	args = appendParameters(args, tags, target.buildPkgs...)
 	runPrint(goCmd, args...)
 }
 
-func appendParameters(args []string, tags []string, target target) []string {
+func setBuildEnvVars() {
+	os.Setenv("GOOS", goos)
+	os.Setenv("GOARCH", goarch)
+	os.Setenv("CC", cc)
+	if os.Getenv("CGO_ENABLED") == "" {
+		switch goos {
+		case "darwin", "solaris":
+		default:
+			os.Setenv("CGO_ENABLED", "0")
+		}
+	}
+}
+
+func appendParameters(args []string, tags []string, pkgs ...string) []string {
 	if pkgdir != "" {
 		args = append(args, "-pkgdir", pkgdir)
 	}
@@ -439,33 +542,31 @@ func appendParameters(args []string, tags []string, target target) []string {
 
 	if !debugBinary {
 		// Regular binaries get version tagged and skip some debug symbols
-		args = append(args, "-ldflags", ldflags())
+		args = append(args, "-ldflags", ldflags(tags))
 	} else {
 		// -gcflags to disable optimizations and inlining. Skip -ldflags
 		// because `Could not launch program: decoding dwarf section info at
 		// offset 0x0: too short` on 'dlv exec ...' see
-		// https://github.com/derekparker/delve/issues/79
+		// https://github.com/go-delve/delve/issues/79
 		args = append(args, "-gcflags", "-N -l")
 	}
 
-	return append(args, target.buildPkg)
+	return append(args, pkgs...)
 }
 
-func buildTar(target target) {
+func buildTar(target target, tags []string) {
 	name := archiveName(target)
 	filename := name + ".tar.gz"
 
-	var tags []string
+	for _, tag := range tags {
-	if noupgrade {
+		if tag == "noupgrade" {
-		tags = []string{"noupgrade"}
+			name += "-noupgrade"
-		name += "-noupgrade"
+			break
+		}
 	}
 
 	build(target, tags)
-
+	codesign(target)
-	if goos == "darwin" {
-		macosCodesign(target.BinaryName())
-	}
 
 	for i := range target.archiveFiles {
 		target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.BinaryName(), 1)
@@ -477,21 +578,19 @@ func buildTar(target target) {
 	fmt.Println(filename)
 }
 
-func buildZip(target target) {
+func buildZip(target target, tags []string) {
 	name := archiveName(target)
 	filename := name + ".zip"
 
-	var tags []string
+	for _, tag := range tags {
-	if noupgrade {
+		if tag == "noupgrade" {
-		tags = []string{"noupgrade"}
+			name += "-noupgrade"
-		name += "-noupgrade"
+			break
+		}
 	}
 
 	build(target, tags)
-
+	codesign(target)
-	if goos == "windows" {
-		windowsCodesign(target.BinaryName())
-	}
 
 	for i := range target.archiveFiles {
 		target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.BinaryName(), 1)
@@ -555,79 +654,58 @@ func buildDeb(target target) {
 	for _, dep := range target.debdeps {
 		args = append(args, "-d", dep)
 	}
+	for _, service := range target.systemdServices {
+		args = append(args, "--deb-systemd", service)
+	}
 	if target.debpost != "" {
 		args = append(args, "--after-upgrade", target.debpost)
 	}
+	if target.debpre != "" {
+		args = append(args, "--before-install", target.debpre)
+	}
 	runPrint("fpm", args...)
 }
 
-func buildSnap(target target) {
+func shouldBuildSyso(dir string) (string, error) {
-	os.RemoveAll("snap")
+	type M map[string]interface{}
-
+	version := getVersion()
-	tmpl, err := template.ParseFiles("snapcraft.yaml.template")
+	version = strings.TrimPrefix(version, "v")
-	if err != nil {
+	major, minor, patch := semanticVersion()
-		log.Fatal(err)
+	bs, err := json.Marshal(M{
-	}
+		"FixedFileInfo": M{
-	f, err := os.Create("snapcraft.yaml")
+			"FileVersion": M{
-	defer f.Close()
+				"Major": major,
-	if err != nil {
+				"Minor": minor,
-		log.Fatal(err)
+				"Patch": patch,
-	}
+			},
-
+			"ProductVersion": M{
-	snaparch := goarch
+				"Major": major,
-	if snaparch == "armhf" {
+				"Minor": minor,
-		goarch = "arm"
+				"Patch": patch,
-	} else if snaparch == "i386" {
+			},
-		goarch = "386"
+		},
-	}
+		"StringFileInfo": M{
-	snapver := version
+			"CompanyName":      "The Syncthing Authors",
-	if strings.HasPrefix(snapver, "v") {
+			"FileDescription":  "Syncthing - Open Source Continuous File Synchronization",
-		snapver = snapver[1:]
+			"FileVersion":      version,
-	}
+			"InternalName":     "syncthing",
-	snapgrade := "devel"
+			"LegalCopyright":   "The Syncthing Authors",
-	if matched, _ := regexp.MatchString(`^\d+\.\d+\.\d+(-rc.\d+)?$`, snapver); matched {
+			"OriginalFilename": "syncthing",
-		snapgrade = "stable"
+			"ProductName":      "Syncthing",
-	}
+			"ProductVersion":   version,
-	err = tmpl.Execute(f, map[string]string{
+		},
-		"Version":            snapver,
+		"IconPath": "assets/logo.ico",
-		"HostArchitecture":   runtime.GOARCH,
-		"TargetArchitecture": snaparch,
-		"Grade":              snapgrade,
 	})
 	if err != nil {
-		log.Fatal(err)
+		return "", err
 	}
-	runPrint("snapcraft", "clean")
-	build(target, []string{"noupgrade"})
-	runPrint("snapcraft")
-}
 
-func shouldBuildSyso(dir string) (string, error) {
 	jsonPath := filepath.Join(dir, "versioninfo.json")
-	file, err := os.Create(filepath.Join(dir, "versioninfo.json"))
+	err = ioutil.WriteFile(jsonPath, bs, 0644)
 	if err != nil {
 		return "", errors.New("failed to create " + jsonPath + ": " + err.Error())
 	}
 
-	major, minor, patch, build := semanticVersion()
-	fmt.Fprintf(file, `{
-    "FixedFileInfo": {
-        "FileVersion": {
-            "Major": %s,
-            "Minor": %s,
-            "Patch": %s,
-            "Build": %s
-        }
-    },
-    "StringFileInfo": {
-        "FileDescription": "Open Source Continuous File Synchronization",
-        "LegalCopyright": "The Syncthing Authors",
-        "ProductVersion": "%s",
-        "ProductName": "Syncthing"
-    },
-    "IconPath": "assets/logo.ico"
-}`, major, minor, patch, build, getVersion())
-	file.Close()
 	defer func() {
 		if err := os.Remove(jsonPath); err != nil {
 			log.Printf("Warning: unable to remove generated %s: %v. Please remove it manually.", jsonPath, err)
@@ -690,6 +768,7 @@ func listFiles(dir string) []string {
 		if err != nil {
 			return err
 		}
+
 		if fi.Mode().IsRegular() {
 			res = append(res, path)
 		}
@@ -700,15 +779,50 @@ func listFiles(dir string) []string {
 
 func rebuildAssets() {
 	os.Setenv("SOURCE_DATE_EPOCH", fmt.Sprint(buildStamp()))
-	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/auto", "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto")
+	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/api/auto", "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto")
 }
 
 func lazyRebuildAssets() {
-	if shouldRebuildAssets("lib/auto/gui.files.go", "gui") || shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.files.go", "cmd/strelaypoolsrv/auto/gui") {
+	shouldRebuild := shouldRebuildAssets("lib/api/auto/gui.files.go", "gui") ||
+		shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.files.go", "cmd/strelaypoolsrv/gui")
+
+	if withNextGenGUI {
+		shouldRebuild = buildNextGenGUI() || shouldRebuild
+	}
+
+	if shouldRebuild {
 		rebuildAssets()
 	}
 }
 
+func buildNextGenGUI() bool {
+	// Check if we need to run the npm process, and if so also set the flag
+	// to rebuild Go assets afterwards. The index.html is regenerated every
+	// time by the build process. This assumes the new GUI ends up in
+	// next-gen-gui/dist/next-gen-gui.
+
+	if !shouldRebuildAssets("gui/next-gen-gui/index.html", "next-gen-gui") {
+		// The GUI is up to date.
+		return false
+	}
+
+	runPrintInDir("next-gen-gui", "npm", "install")
+	runPrintInDir("next-gen-gui", "npm", "run", "build", "--", "--prod", "--subresource-integrity")
+
+	rmr("gui/tech-ui")
+
+	for _, src := range listFiles("next-gen-gui/dist") {
+		rel, _ := filepath.Rel("next-gen-gui/dist", src)
+		dst := filepath.Join("gui", rel)
+		if err := copyFile(src, dst, 0644); err != nil {
+			fmt.Println("copy:", err)
+			os.Exit(1)
+		}
+	}
+
+	return true
+}
+
 func shouldRebuildAssets(target, srcdir string) bool {
 	info, err := os.Stat(target)
 	if err != nil {
@@ -736,16 +850,40 @@ func shouldRebuildAssets(target, srcdir string) bool {
 }
 
 func proto() {
-	runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", gogoProtoVersion))
+	pv := protobufVersion()
+	dependencyRepos = append(dependencyRepos,
+		dependencyRepo{path: "protobuf", repo: "https://github.com/gogo/protobuf.git", commit: pv},
+	)
+
+	runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
 	os.MkdirAll("repos", 0755)
 	for _, dep := range dependencyRepos {
 		path := filepath.Join("repos", dep.path)
 		if _, err := os.Stat(path); err != nil {
 			runPrintInDir("repos", "git", "clone", dep.repo, dep.path)
-			runPrintInDir(path, "git", "checkout", dep.commit)
+		} else {
+			runPrintInDir(path, "git", "fetch")
 		}
+		runPrintInDir(path, "git", "checkout", dep.commit)
 	}
-	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/stdiscosrv")
+	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/cmd/stdiscosrv")
+	runPrint(goCmd, "generate", "proto/generate.go")
+}
+
+func testmocks() {
+	runPrint(goCmd, "get", "golang.org/x/tools/cmd/goimports")
+	runPrint(goCmd, "get", "github.com/maxbrunsfeld/counterfeiter/v6")
+	args := []string{
+		"generate",
+		"github.com/syncthing/syncthing/lib/config",
+		"github.com/syncthing/syncthing/lib/connections",
+		"github.com/syncthing/syncthing/lib/discover",
+		"github.com/syncthing/syncthing/lib/events",
+		"github.com/syncthing/syncthing/lib/logger",
+		"github.com/syncthing/syncthing/lib/model",
+		"github.com/syncthing/syncthing/lib/protocol",
+	}
+	runPrint(goCmd, args...)
 }
 
 func translate() {
@@ -764,18 +902,17 @@ func transifex() {
 	runPrint(goCmd, "run", "../../../../script/transifexdl.go")
 }
 
-func ldflags() string {
+func ldflags(tags []string) string {
-	sep := '='
+	b := new(strings.Builder)
-	if goVersion > 0 && goVersion < 1.5 {
-		sep = ' '
-	}
-
-	b := new(bytes.Buffer)
 	b.WriteString("-w")
-	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Version%c%s", sep, version)
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Version=%s", version)
-	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp%c%d", sep, buildStamp())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp=%d", buildStamp())
-	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User%c%s", sep, buildUser())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User=%s", buildUser())
-	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Host%c%s", sep, buildHost())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Host=%s", buildHost())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Tags=%s", strings.Join(tags, ","))
+	if v := os.Getenv("EXTRA_LDFLAGS"); v != "" {
+		fmt.Fprintf(b, " %s", v)
+	}
 	return b.String()
 }
 
@@ -789,13 +926,7 @@ func rmr(paths ...string) {
 }
 
 func getReleaseVersion() (string, error) {
-	fd, err := os.Open("RELEASE")
+	bs, err := ioutil.ReadFile("RELEASE")
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-
-	bs, err := ioutil.ReadAll(fd)
 	if err != nil {
 		return "", err
 	}
@@ -803,15 +934,39 @@ func getReleaseVersion() (string, error) {
 }
 
 func getGitVersion() (string, error) {
-	v, err := runError("git", "describe", "--always", "--dirty")
+	// The current version as Git sees it
+	bs, err := runError("git", "describe", "--always", "--dirty", "--abbrev=8")
 	if err != nil {
 		return "", err
 	}
-	v = versionRe.ReplaceAllFunc(v, func(s []byte) []byte {
+	vcur := string(bs)
-		s[0] = '+'
+
-		return s
+	// The closest current tag name
-	})
+	bs, err = runError("git", "describe", "--always", "--abbrev=0")
-	return string(v), nil
+	if err != nil {
+		return "", err
+	}
+	v0 := string(bs)
+
+	// To be more semantic-versionish and ensure proper ordering in our
+	// upgrade process, we make sure there's only one hypen in the version.
+
+	versionRe := regexp.MustCompile(`-([0-9]{1,3}-g[0-9a-f]{5,10}(-dirty)?)`)
+	if m := versionRe.FindStringSubmatch(vcur); len(m) > 0 {
+		suffix := strings.ReplaceAll(m[1], "-", ".")
+
+		if strings.Contains(v0, "-") {
+			// We're based of a tag with a prerelease string. We can just
+			// add our dev stuff directly.
+			return fmt.Sprintf("%s.dev.%s", v0, suffix), nil
+		}
+
+		// We're based on a release version. We need to bump the patch
+		// version and then add a -dev prerelease string.
+		next := nextPatchVersion(v0)
+		return fmt.Sprintf("%s-dev.%s", next, suffix), nil
+	}
+	return vcur, nil
 }
 
 func getVersion() string {
@@ -832,13 +987,18 @@ func getVersion() string {
 	return "unknown-dev"
 }
 
-func semanticVersion() (major, minor, patch, build string) {
+func semanticVersion() (major, minor, patch int) {
-	r := regexp.MustCompile(`v(?P<Major>\d+)\.(?P<Minor>\d+).(?P<Patch>\d+).*\+(?P<CommitsAhead>\d+)`)
+	r := regexp.MustCompile(`v(\d+)\.(\d+).(\d+)`)
 	matches := r.FindStringSubmatch(getVersion())
-	if len(matches) != 5 {
+	if len(matches) != 4 {
-		return "0", "0", "0", "0"
+		return 0, 0, 0
 	}
-	return matches[1], matches[2], matches[3], matches[4]
+
+	var ints [3]int
+	for i, s := range matches[1:] {
+		ints[i], _ = strconv.Atoi(s)
+	}
+	return ints[0], ints[1], ints[2]
 }
 
 func getBranchSuffix() string {
@@ -877,7 +1037,7 @@ func getBranchSuffix() string {
 
 	branch = parts[len(parts)-1]
 	switch branch {
-	case "master", "release":
+	case "master", "release", "main":
 		// these are not special
 		return ""
 	}
@@ -1134,6 +1294,15 @@ func zipFile(out string, files []archiveFile) {
 	}
 }
 
+func codesign(target target) {
+	switch goos {
+	case "windows":
+		windowsCodesign(target.BinaryName())
+	case "darwin":
+		macosCodesign(target.BinaryName())
+	}
+}
+
 func macosCodesign(file string) {
 	if pass := os.Getenv("CODESIGN_KEYCHAIN_PASS"); pass != "" {
 		bs, err := runError("security", "unlock-keychain", "-p", pass)
@@ -1144,7 +1313,7 @@ func macosCodesign(file string) {
 	}
 
 	if id := os.Getenv("CODESIGN_IDENTITY"); id != "" {
-		bs, err := runError("codesign", "-s", id, file)
+		bs, err := runError("codesign", "--options=runtime", "-s", id, file)
 		if err != nil {
 			log.Println("Codesign: signing failed:", string(bs))
 			return
@@ -1232,3 +1401,78 @@ func (t target) BinaryName() string {
 	}
 	return t.binaryName
 }
+
+func protobufVersion() string {
+	bs, err := runError(goCmd, "list", "-f", "{{.Version}}", "-m", "github.com/gogo/protobuf")
+	if err != nil {
+		log.Fatal("Getting protobuf version:", err)
+	}
+	return string(bs)
+}
+
+func currentAndLatestVersions(n int) ([]string, error) {
+	bs, err := runError("git", "tag", "--sort", "taggerdate")
+	if err != nil {
+		return nil, err
+	}
+
+	lines := strings.Split(string(bs), "\n")
+	reverseStrings(lines)
+
+	// The one at the head is the latest version. We always keep that one.
+	// Then we filter out remaining ones with dashes (pre-releases etc).
+
+	latest := lines[:1]
+	nonPres := filterStrings(lines[1:], func(s string) bool { return !strings.Contains(s, "-") })
+	vers := append(latest, nonPres...)
+	return vers[:n], nil
+}
+
+func reverseStrings(ss []string) {
+	for i := 0; i < len(ss)/2; i++ {
+		ss[i], ss[len(ss)-1-i] = ss[len(ss)-1-i], ss[i]
+	}
+}
+
+func filterStrings(ss []string, op func(string) bool) []string {
+	n := ss[:0]
+	for _, s := range ss {
+		if op(s) {
+			n = append(n, s)
+		}
+	}
+	return n
+}
+
+func tagMessage(tag string) (string, error) {
+	hash, err := runError("git", "rev-parse", tag)
+	if err != nil {
+		return "", err
+	}
+	obj, err := runError("git", "cat-file", "-p", string(hash))
+	if err != nil {
+		return "", err
+	}
+	return trimTagMessage(string(obj), tag), nil
+}
+
+func trimTagMessage(msg, tag string) string {
+	firstBlank := strings.Index(msg, "\n\n")
+	if firstBlank > 0 {
+		msg = msg[firstBlank+2:]
+	}
+	msg = strings.TrimPrefix(msg, tag)
+	beginSig := strings.Index(msg, "-----BEGIN PGP")
+	if beginSig > 0 {
+		msg = msg[:beginSig]
+	}
+	return strings.TrimSpace(msg)
+}
+
+func nextPatchVersion(ver string) string {
+	parts := strings.SplitN(ver, "-", 2)
+	digits := strings.Split(parts[0], ".")
+	n, _ := strconv.Atoi(digits[len(digits)-1])
+	digits[len(digits)-1] = strconv.Itoa(n + 1)
+	return strings.Join(digits, ".")
+}

build.ps1

@@ -0,0 +1,20 @@
+function build {
+    go run build.go @args
+}
+
+$cmd, $rest = $args
+switch ($cmd) {
+    "test" {
+        $env:LOGGER_DISCARD=1
+        build test
+    }
+
+    "bench" {
+        $env:LOGGER_DISCARD=1
+        build bench
+    }
+
+    default {
+        build @rest
+    }
+}

build.sh

@@ -2,8 +2,6 @@
 set -euo pipefail
 IFS=$'\n\t'
 
-STTRACE=${STTRACE:-}
-
 script() {
 	name="$1"
 	shift
@@ -15,88 +13,23 @@ build() {
 }
 
 case "${1:-default}" in
-	default)
-		build
-		;;
-
-	clean)
-		build "$@"
-		;;
-
-	tar)
-		build "$@"
-		;;
-
-	assets)
-		build "$@"
-		;;
-
-	xdr)
-		build "$@"
-		;;
-
-	translate)
-		build "$@"
-		;;
-
-	deb)
-		build "$@"
-		;;
-
-	setup)
-		build "$@"
-		;;
-
 	test)
 		LOGGER_DISCARD=1 build test
 		;;
 
 	bench)
-		LOGGER_DISCARD=1 build bench | script benchfilter
+		LOGGER_DISCARD=1 build bench
 		;;
 
 	prerelease)
-		go run script/authors.go
+		script authors
 		build transifex
 		pushd man ; ./refresh.sh ; popd
 		git add -A gui man AUTHORS
 		git commit -m 'gui, man, authors: Update docs, translations, and contributors'
 		;;
 
-	noupgrade)
-		build -no-upgrade tar
-		;;
-
-	all)
-		platforms=(
-			darwin-amd64 dragonfly-amd64 freebsd-amd64 linux-amd64 netbsd-amd64 openbsd-amd64 solaris-amd64 windows-amd64
-			freebsd-386 linux-386 netbsd-386 openbsd-386 windows-386
-			linux-arm linux-arm64 linux-ppc64 linux-ppc64le
-		)
-
-		for plat in "${platforms[@]}"; do
-			echo Building "$plat"
-
-			goos="${plat%-*}"
-			goarch="${plat#*-}"
-			dist="tar"
-
-			if [[ $goos == "windows" ]]; then
-				dist="zip"
-			fi
-
-			build -goos "$goos" -goarch "$goarch" "$dist"
-			echo
-		done
-		;;
-
-	test-xunit)
-
-		(GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -v -race ./lib/... ./cmd/... || true) > tests.out
-		go2xunit -output tests.xml -fail < tests.out
-		;;
-
 	*)
-		echo "Unknown build command $1"
+		build "$@"
 		;;
 esac

cmd/stbench/main.go

@@ -1,143 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// This doesn't build on Windows due to the Rusage stuff.
-
-// +build !windows
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"runtime"
-	"syscall"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/rc"
-)
-
-var homeDir = "h1"
-var syncthingBin = "./bin/syncthing"
-var test = "scan"
-
-func main() {
-	flag.StringVar(&homeDir, "home", homeDir, "Home directory location")
-	flag.StringVar(&syncthingBin, "bin", syncthingBin, "Binary location")
-	flag.StringVar(&test, "test", test, "Test to run")
-	flag.Parse()
-
-	switch test {
-	case "scan":
-		// scan measures the resource usage required to perform the initial
-		// scan, without cleaning away the database first.
-		testScan()
-	}
-}
-
-// testScan starts a process and reports on the resource usage required to
-// perform the initial scan.
-func testScan() {
-	log.Println("Starting...")
-	p := rc.NewProcess("127.0.0.1:8081")
-	if err := p.Start(syncthingBin, "-home", homeDir, "-no-browser"); err != nil {
-		log.Println(err)
-		return
-	}
-	defer p.Stop()
-
-	wallTime := awaitScanComplete(p)
-
-	report(p, wallTime)
-}
-
-// awaitScanComplete waits for a folder to transition idle->scanning and
-// then scanning->idle and returns the time taken for the scan.
-func awaitScanComplete(p *rc.Process) time.Duration {
-	log.Println("Awaiting scan completion...")
-	var t0, t1 time.Time
-	lastEvent := 0
-loop:
-	for {
-		evs, err := p.Events(lastEvent)
-		if err != nil {
-			continue
-		}
-
-		for _, ev := range evs {
-			if ev.Type == "StateChanged" {
-				data := ev.Data.(map[string]interface{})
-				log.Println(ev)
-
-				if data["to"].(string) == "scanning" {
-					t0 = ev.Time
-					continue
-				}
-
-				if !t0.IsZero() && data["to"].(string) == "idle" {
-					t1 = ev.Time
-					break loop
-				}
-			}
-			lastEvent = ev.ID
-		}
-
-		time.Sleep(250 * time.Millisecond)
-	}
-
-	return t1.Sub(t0)
-}
-
-// report stops the given process and reports on its resource usage in two
-// ways: human readable to stderr, and CSV to stdout.
-func report(p *rc.Process, wallTime time.Duration) {
-	sv, err := p.SystemVersion()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	ss, err := p.SystemStatus()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	proc, err := p.Stop()
-	if err != nil {
-		return
-	}
-
-	rusage, ok := proc.SysUsage().(*syscall.Rusage)
-	if !ok {
-		return
-	}
-
-	log.Println("Version:", sv.Version)
-	log.Println("Alloc:", ss.Alloc/1024, "KiB")
-	log.Println("Sys:", ss.Sys/1024, "KiB")
-	log.Println("Goroutines:", ss.Goroutines)
-	log.Println("Wall time:", wallTime)
-	log.Println("Utime:", time.Duration(rusage.Utime.Nano()))
-	log.Println("Stime:", time.Duration(rusage.Stime.Nano()))
-	if runtime.GOOS == "darwin" {
-		// Darwin reports in bytes, Linux seems to report in KiB even
-		// though the manpage says otherwise.
-		rusage.Maxrss /= 1024
-	}
-	log.Println("MaxRSS:", rusage.Maxrss, "KiB")
-
-	fmt.Printf("%s,%d,%d,%d,%.02f,%.02f,%.02f,%d\n",
-		sv.Version,
-		ss.Alloc/1024,
-		ss.Sys/1024,
-		ss.Goroutines,
-		wallTime.Seconds(),
-		time.Duration(rusage.Utime.Nano()).Seconds(),
-		time.Duration(rusage.Stime.Nano()).Seconds(),
-		rusage.Maxrss)
-}

cmd/stcli/main.go

@@ -1,192 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bufio"
-	"crypto/tls"
-	"encoding/json"
-	"flag"
-	"log"
-	"os"
-	"reflect"
-	"strings"
-
-	"github.com/AudriusButkevicius/recli"
-	"github.com/flynn-archive/go-shlex"
-	"github.com/mattn/go-isatty"
-	"github.com/pkg/errors"
-	"github.com/syncthing/syncthing/lib/build"
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/locations"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/urfave/cli"
-)
-
-func main() {
-	// This is somewhat a hack around a chicken and egg problem.
-	// We need to set the home directory and potentially other flags to know where the syncthing instance is running
-	// in order to get it's config ... which we then use to construct the actual CLI ... at which point it's too late
-	// to add flags there...
-	homeBaseDir := locations.GetBaseDir(locations.ConfigBaseDir)
-	guiCfg := config.GUIConfiguration{}
-
-	flags := flag.NewFlagSet("", flag.ContinueOnError)
-	flags.StringVar(&guiCfg.RawAddress, "gui-address", guiCfg.RawAddress, "Override GUI address (e.g. \"http://192.0.2.42:8443\")")
-	flags.StringVar(&guiCfg.APIKey, "gui-apikey", guiCfg.APIKey, "Override GUI API key")
-	flags.StringVar(&homeBaseDir, "home", homeBaseDir, "Set configuration directory")
-
-	// Implement the same flags at the lower CLI, with the same default values (pre-parse), but do nothing with them.
-	// This is so that we could reuse os.Args
-	fakeFlags := []cli.Flag{
-		cli.StringFlag{
-			Name:  "gui-address",
-			Value: guiCfg.RawAddress,
-			Usage: "Override GUI address (e.g. \"http://192.0.2.42:8443\")",
-		},
-		cli.StringFlag{
-			Name:  "gui-apikey",
-			Value: guiCfg.APIKey,
-			Usage: "Override GUI API key",
-		},
-		cli.StringFlag{
-			Name:  "home",
-			Value: homeBaseDir,
-			Usage: "Set configuration directory",
-		},
-	}
-
-	// Do not print usage of these flags, and ignore errors as this can't understand plenty of things
-	flags.Usage = func() {}
-	_ = flags.Parse(os.Args[1:])
-
-	// Now if the API key and address is not provided (we are not connecting to a remote instance),
-	// try to rip it out of the config.
-	if guiCfg.RawAddress == "" && guiCfg.APIKey == "" {
-		// Update the base directory
-		err := locations.SetBaseDir(locations.ConfigBaseDir, homeBaseDir)
-		if err != nil {
-			log.Fatal(errors.Wrap(err, "setting home"))
-		}
-
-		// Load the certs and get the ID
-		cert, err := tls.LoadX509KeyPair(
-			locations.Get(locations.CertFile),
-			locations.Get(locations.KeyFile),
-		)
-		if err != nil {
-			log.Fatal(errors.Wrap(err, "reading device ID"))
-		}
-
-		myID := protocol.NewDeviceID(cert.Certificate[0])
-
-		// Load the config
-		cfg, err := config.Load(locations.Get(locations.ConfigFile), myID)
-		if err != nil {
-			log.Fatalln(errors.Wrap(err, "loading config"))
-		}
-
-		guiCfg = cfg.GUI()
-	} else if guiCfg.Address() == "" || guiCfg.APIKey == "" {
-		log.Fatalln("Both -gui-address and -gui-apikey should be specified")
-	}
-
-	if guiCfg.Address() == "" {
-		log.Fatalln("Could not find GUI Address")
-	}
-
-	if guiCfg.APIKey == "" {
-		log.Fatalln("Could not find GUI API key")
-	}
-
-	client := getClient(guiCfg)
-
-	cfg, err := getConfig(client)
-	original := cfg.Copy()
-	if err != nil {
-		log.Fatalln(errors.Wrap(err, "getting config"))
-	}
-
-	// Copy the config and set the default flags
-	recliCfg := recli.DefaultConfig
-	recliCfg.IDTag.Name = "xml"
-	recliCfg.SkipTag.Name = "json"
-
-	commands, err := recli.New(recliCfg).Construct(&cfg)
-	if err != nil {
-		log.Fatalln(errors.Wrap(err, "config reflect"))
-	}
-
-	// Construct the actual CLI
-	app := cli.NewApp()
-	app.Name = "stcli"
-	app.HelpName = app.Name
-	app.Author = "The Syncthing Authors"
-	app.Usage = "Syncthing command line interface"
-	app.Version = strings.Replace(build.LongVersion, "syncthing", app.Name, 1)
-	app.Flags = fakeFlags
-	app.Metadata = map[string]interface{}{
-		"client": client,
-	}
-	app.Commands = []cli.Command{
-		{
-			Name:        "config",
-			HideHelp:    true,
-			Usage:       "Configuration modification command group",
-			Subcommands: commands,
-		},
-		showCommand,
-		operationCommand,
-		errorsCommand,
-	}
-
-	tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
-	if !tty {
-		// Not a TTY, consume from stdin
-		scanner := bufio.NewScanner(os.Stdin)
-		for scanner.Scan() {
-			input, err := shlex.Split(scanner.Text())
-			if err != nil {
-				log.Fatalln(errors.Wrap(err, "parsing input"))
-			}
-			if len(input) == 0 {
-				continue
-			}
-			err = app.Run(append(os.Args, input...))
-			if err != nil {
-				log.Fatalln(err)
-			}
-		}
-		err = scanner.Err()
-		if err != nil {
-			log.Fatalln(err)
-		}
-	} else {
-		err = app.Run(os.Args)
-		if err != nil {
-			log.Fatalln(err)
-		}
-	}
-
-	if !reflect.DeepEqual(cfg, original) {
-		body, err := json.MarshalIndent(cfg, "", "  ")
-		if err != nil {
-			log.Fatalln(err)
-		}
-		resp, err := client.Post("system/config", string(body))
-		if err != nil {
-			log.Fatalln(err)
-		}
-		if resp.StatusCode != 200 {
-			body, err := responseToBArray(resp)
-			if err != nil {
-				log.Fatalln(err)
-			}
-			log.Fatalln(string(body))
-		}
-	}
-}

cmd/stcompdirs/main.go

@@ -7,7 +7,6 @@
 package main
 
 import (
-	"crypto/md5"
 	"errors"
 	"flag"
 	"fmt"
@@ -15,6 +14,8 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 func main() {
@@ -60,7 +61,7 @@ func compareDirectories(dirs ...string) error {
 				} else if res[i].name > res[0].name {
 					return fmt.Errorf("%s missing %v (present in %s)", dirs[i], res[0], dirs[0])
 				}
-				return fmt.Errorf("Mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
+				return fmt.Errorf("mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
 			}
 		}
 
@@ -74,7 +75,7 @@ type fileInfo struct {
 	name string
 	mode os.FileMode
 	mod  int64
-	hash [16]byte
+	hash [sha256.Size]byte
 }
 
 func (f fileInfo) String() string {
@@ -106,11 +107,7 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 			if err != nil {
 				return err
 			}
-			h := md5.New()
+			f.hash = sha256.Sum256([]byte(tgt))
-			h.Write([]byte(tgt))
-			hash := h.Sum(nil)
-
-			copy(f.hash[:], hash)
 		} else if info.IsDir() {
 			f = fileInfo{
 				name: rn,
@@ -123,7 +120,7 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 				mode: info.Mode(),
 				mod:  info.ModTime().Unix(),
 			}
-			sum, err := md5file(path)
+			sum, err := sha256file(path)
 			if err != nil {
 				return err
 			}
@@ -150,14 +147,14 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 	return errc
 }
 
-func md5file(fname string) (hash [16]byte, err error) {
+func sha256file(fname string) (hash [sha256.Size]byte, err error) {
 	f, err := os.Open(fname)
 	if err != nil {
 		return
 	}
 	defer f.Close()
 
-	h := md5.New()
+	h := sha256.New()
 	io.Copy(h, f)
 	hb := h.Sum(nil)
 	copy(hash[:], hb)

cmd/stcrashreceiver/main.go

@@ -0,0 +1,118 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Command stcrashreceiver is a trivial HTTP server that allows two things:
+//
+// - uploading files (crash reports) named like a SHA256 hash using a PUT request
+// - checking whether such file exists using a HEAD request
+//
+// Typically this should be deployed behind something that manages HTTPS.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/sha256"
+	"github.com/syncthing/syncthing/lib/ur"
+
+	raven "github.com/getsentry/raven-go"
+)
+
+const maxRequestSize = 1 << 20 // 1 MiB
+
+func main() {
+	dir := flag.String("dir", ".", "Directory to store reports in")
+	dsn := flag.String("dsn", "", "Sentry DSN")
+	listen := flag.String("listen", ":22039", "HTTP listen address")
+	flag.Parse()
+
+	mux := http.NewServeMux()
+
+	cr := &crashReceiver{
+		dir: *dir,
+		dsn: *dsn,
+	}
+	mux.Handle("/", cr)
+
+	if *dsn != "" {
+		mux.HandleFunc("/newcrash/failure", handleFailureFn(*dsn))
+	}
+
+	log.SetOutput(os.Stdout)
+	if err := http.ListenAndServe(*listen, mux); err != nil {
+		log.Fatalln("HTTP serve:", err)
+	}
+}
+
+func handleFailureFn(dsn string) func(w http.ResponseWriter, req *http.Request) {
+	return func(w http.ResponseWriter, req *http.Request) {
+		lr := io.LimitReader(req.Body, maxRequestSize)
+		bs, err := ioutil.ReadAll(lr)
+		req.Body.Close()
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+
+		var reports []ur.FailureReport
+		err = json.Unmarshal(bs, &reports)
+		if err != nil {
+			http.Error(w, err.Error(), 400)
+			return
+		}
+		if len(reports) == 0 {
+			// Shouldn't happen
+			log.Printf("Got zero failure reports")
+			return
+		}
+
+		version, err := parseVersion(reports[0].Version)
+		if err != nil {
+			http.Error(w, err.Error(), 400)
+			return
+		}
+		for _, r := range reports {
+			pkt := packet(version, "failure")
+			pkt.Message = r.Description
+			pkt.Extra = raven.Extra{
+				"count": r.Count,
+			}
+			pkt.Fingerprint = []string{r.Description}
+
+			if err := sendReport(dsn, pkt, userIDFor(req)); err != nil {
+				log.Println("Failed to send failure report:", err)
+			} else {
+				log.Println("Sent failure report:", r.Description)
+			}
+		}
+	}
+}
+
+// userIDFor returns a string we can use as the user ID for the purpose of
+// counting affected users. It's the truncated hash of a salt, the user
+// remote IP, and the current month.
+func userIDFor(req *http.Request) string {
+	addr := req.RemoteAddr
+	if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
+		addr = fwd
+	}
+	if host, _, err := net.SplitHostPort(addr); err == nil {
+		addr = host
+	}
+	now := time.Now().Format("200601")
+	salt := "stcrashreporter"
+	hash := sha256.Sum256([]byte(salt + addr + now))
+	return fmt.Sprintf("%x", hash[:8])
+}

cmd/stcrashreceiver/sentry.go

@@ -0,0 +1,255 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"regexp"
+	"strings"
+	"sync"
+
+	raven "github.com/getsentry/raven-go"
+	"github.com/maruel/panicparse/stack"
+)
+
+const reportServer = "https://crash.syncthing.net/report/"
+
+var loader = newGithubSourceCodeLoader()
+
+func init() {
+	raven.SetSourceCodeLoader(loader)
+}
+
+var (
+	clients    = make(map[string]*raven.Client)
+	clientsMut sync.Mutex
+)
+
+func sendReport(dsn string, pkt *raven.Packet, userID string) error {
+	pkt.Interfaces = append(pkt.Interfaces, &raven.User{ID: userID})
+
+	clientsMut.Lock()
+	defer clientsMut.Unlock()
+
+	cli, ok := clients[dsn]
+	if !ok {
+		var err error
+		cli, err = raven.New(dsn)
+		if err != nil {
+			return err
+		}
+		clients[dsn] = cli
+	}
+
+	// The client sets release and such on the packet before sending, in the
+	// misguided idea that it knows this better than than the packet we give
+	// it. So we copy the values from the packet to the client first...
+	cli.SetRelease(pkt.Release)
+	cli.SetEnvironment(pkt.Environment)
+
+	defer cli.Wait()
+	_, errC := cli.Capture(pkt, nil)
+	return <-errC
+}
+
+func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
+	parts := bytes.SplitN(report, []byte("\n"), 2)
+	if len(parts) != 2 {
+		return nil, errors.New("no first line")
+	}
+
+	version, err := parseVersion(string(parts[0]))
+	if err != nil {
+		return nil, err
+	}
+	report = parts[1]
+
+	foundPanic := false
+	var subjectLine []byte
+	for {
+		parts = bytes.SplitN(report, []byte("\n"), 2)
+		if len(parts) != 2 {
+			return nil, errors.New("no panic line found")
+		}
+
+		line := parts[0]
+		report = parts[1]
+
+		if foundPanic {
+			// The previous line was our "Panic at ..." header. We are now
+			// at the beginning of the real panic trace and this is our
+			// subject line.
+			subjectLine = line
+			break
+		} else if bytes.HasPrefix(line, []byte("Panic at")) {
+			foundPanic = true
+		}
+	}
+
+	r := bytes.NewReader(report)
+	ctx, err := stack.ParseDump(r, ioutil.Discard, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// Lock the source code loader to the version we are processing here.
+	if version.commit != "" {
+		// We have a commit hash, so we know exactly which source to use
+		loader.LockWithVersion(version.commit)
+	} else if strings.HasPrefix(version.tag, "v") {
+		// Lets hope the tag is close enough
+		loader.LockWithVersion(version.tag)
+	} else {
+		// Last resort
+		loader.LockWithVersion("main")
+	}
+	defer loader.Unlock()
+
+	var trace raven.Stacktrace
+	for _, gr := range ctx.Goroutines {
+		if gr.First {
+			trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
+			for i, sc := range gr.Stack.Calls {
+				trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name(), sc.SrcPath, sc.Line, 3, nil)
+			}
+			break
+		}
+	}
+
+	pkt := packet(version, "crash")
+	pkt.Message = string(subjectLine)
+	pkt.Extra = raven.Extra{
+		"url": reportServer + path,
+	}
+	pkt.Interfaces = []raven.Interface{&trace}
+	pkt.Fingerprint = crashReportFingerprint(pkt.Message)
+
+	return pkt, nil
+}
+
+var (
+	indexRe          = regexp.MustCompile(`\[[-:0-9]+\]`)
+	sizeRe           = regexp.MustCompile(`(length|capacity) [0-9]+`)
+	ldbPosRe         = regexp.MustCompile(`(\(pos=)([0-9]+)\)`)
+	ldbChecksumRe    = regexp.MustCompile(`(want=0x)([a-z0-9]+)( got=0x)([a-z0-9]+)`)
+	ldbFileRe        = regexp.MustCompile(`(\[file=)([0-9]+)(\.ldb\])`)
+	ldbInternalKeyRe = regexp.MustCompile(`(internal key ")[^"]+(", len=)[0-9]+`)
+	ldbPathRe        = regexp.MustCompile(`(open|write|read) .+[\\/].+[\\/]index[^\\/]+[\\/][^\\/]+: `)
+)
+
+func crashReportFingerprint(message string) []string {
+	// Do not fingerprint on the stack in case of db corruption or fatal
+	// db io error - where it occurs doesn't matter.
+	orig := message
+	message = ldbPosRe.ReplaceAllString(message, "${1}x)")
+	message = ldbFileRe.ReplaceAllString(message, "${1}x${3}")
+	message = ldbChecksumRe.ReplaceAllString(message, "${1}X${3}X")
+	message = ldbInternalKeyRe.ReplaceAllString(message, "${1}x${2}x")
+	message = ldbPathRe.ReplaceAllString(message, "$1 x: ")
+	if message != orig {
+		return []string{message}
+	}
+
+	message = indexRe.ReplaceAllString(message, "[x]")
+	message = sizeRe.ReplaceAllString(message, "$1 x")
+
+	// {{ default }} is what sentry uses as a fingerprint by default. While
+	// never specified, the docs point at this being some hash derived from the
+	// stack trace. Here we include the filtered panic message on top of that.
+	// https://docs.sentry.io/platforms/go/data-management/event-grouping/sdk-fingerprinting/#basic-example
+	return []string{"{{ default }}", message}
+}
+
+// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]
+var longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
+
+type version struct {
+	version  string   // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
+	tag      string   // "v1.1.4-rc.1"
+	commit   string   // "6aaae618", blank when absent
+	codename string   // "Erbium Earthworm"
+	runtime  string   // "go1.12.5"
+	goos     string   // "darwin"
+	goarch   string   // "amd64"
+	builder  string   // "jb@kvin.kastelo.net"
+	extra    []string // "foo", "bar"
+}
+
+func (v version) environment() string {
+	if v.commit != "" {
+		return "Development"
+	}
+	if strings.Contains(v.tag, "-rc.") {
+		return "Candidate"
+	}
+	if strings.Contains(v.tag, "-") {
+		return "Beta"
+	}
+	return "Stable"
+}
+
+func parseVersion(line string) (version, error) {
+	m := longVersionRE.FindStringSubmatch(line)
+	if len(m) == 0 {
+		return version{}, errors.New("unintelligeble version string")
+	}
+
+	v := version{
+		version:  m[1],
+		codename: m[2],
+		runtime:  m[3],
+		goos:     m[4],
+		goarch:   m[5],
+		builder:  m[6],
+	}
+
+	parts := strings.Split(v.version, "+")
+	v.tag = parts[0]
+	if len(parts) > 1 {
+		fields := strings.Split(parts[1], "-")
+		if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
+			v.commit = fields[1][1:]
+		}
+	}
+
+	if len(m) >= 8 && m[7] != "" {
+		tags := strings.Split(m[7], ",")
+		for i := range tags {
+			tags[i] = strings.TrimSpace(tags[i])
+		}
+		v.extra = tags
+	}
+
+	return v, nil
+}
+
+func packet(version version, reportType string) *raven.Packet {
+	pkt := &raven.Packet{
+		Platform:    "go",
+		Release:     version.tag,
+		Environment: version.environment(),
+		Tags: raven.Tags{
+			raven.Tag{Key: "version", Value: version.version},
+			raven.Tag{Key: "tag", Value: version.tag},
+			raven.Tag{Key: "codename", Value: version.codename},
+			raven.Tag{Key: "runtime", Value: version.runtime},
+			raven.Tag{Key: "goos", Value: version.goos},
+			raven.Tag{Key: "goarch", Value: version.goarch},
+			raven.Tag{Key: "builder", Value: version.builder},
+			raven.Tag{Key: "report_type", Value: reportType},
+		},
+	}
+	if version.commit != "" {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
+	}
+	for _, tag := range version.extra {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: tag, Value: "1"})
+	}
+	return pkt
+}

cmd/stcrashreceiver/sentry_test.go

@@ -0,0 +1,141 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+)
+
+func TestParseVersion(t *testing.T) {
+	cases := []struct {
+		longVersion string
+		parsed      version
+	}{
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+			},
+		},
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+				extra:    []string{"foo", "bar"},
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		v, err := parseVersion(tc.longVersion)
+		if err != nil {
+			t.Errorf("%s\nerror: %v\n", tc.longVersion, err)
+			continue
+		}
+		if fmt.Sprint(v) != fmt.Sprint(tc.parsed) {
+			t.Errorf("%s\nA: %v\nE: %v\n", tc.longVersion, v, tc.parsed)
+		}
+	}
+}
+
+func TestParseReport(t *testing.T) {
+	bs, err := ioutil.ReadFile("_testdata/panic.log")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pkt, err := parseCrashReport("1/2/345", bs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bs, err = pkt.JSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Printf("%s\n", bs)
+}
+
+func TestCrashReportFingerprint(t *testing.T) {
+	cases := []struct {
+		message, exp string
+		ldb          bool
+	}{
+		{
+			message: "panic: leveldb/table: corruption on data-block (pos=51308946): checksum mismatch, want=0xa89f9aa0 got=0xd27cc4c7 [file=004003.ldb]",
+			exp:     "panic: leveldb/table: corruption on data-block (pos=x): checksum mismatch, want=0xX got=0xX [file=x.ldb]",
+			ldb:     true,
+		},
+		{
+			message: "panic: leveldb/table: corruption on table-footer (pos=248): bad magic number [file=001370.ldb]",
+			exp:     "panic: leveldb/table: corruption on table-footer (pos=x): bad magic number [file=x.ldb]",
+			ldb:     true,
+		},
+		{
+			message: "panic: runtime error: slice bounds out of range [4294967283:4194304]",
+			exp:     "panic: runtime error: slice bounds out of range [x]",
+		},
+		{
+			message: "panic: runtime error: slice bounds out of range [-2:]",
+			exp:     "panic: runtime error: slice bounds out of range [x]",
+		},
+		{
+			message: "panic: runtime error: slice bounds out of range [:4294967283] with capacity 32768",
+			exp:     "panic: runtime error: slice bounds out of range [x] with capacity x",
+		},
+		{
+			message: "panic: runtime error: index out of range [0] with length 0",
+			exp:     "panic: runtime error: index out of range [x] with length x",
+		},
+		{
+			message: `panic: leveldb: internal key "\x01", len=1: invalid length`,
+			exp:     `panic: leveldb: internal key "x", len=x: invalid length`,
+			ldb:     true,
+		},
+		{
+			message: `panic: write /var/syncthing/config/index-v0.14.0.db/2732813.log: cannot allocate memory`,
+			exp:     `panic: write x: cannot allocate memory`,
+			ldb:     true,
+		},
+		{
+			message: `panic: filling Blocks: read C:\Users\Serv-Resp-Tizayuca\AppData\Local\Syncthing\index-v0.14.0.db\006561.ldb: Error de datos (comprobación de redundancia cíclica).`,
+			exp:     `panic: filling Blocks: read x: Error de datos (comprobación de redundancia cíclica).`,
+			ldb:     true,
+		},
+	}
+
+	for i, tc := range cases {
+		fingerprint := crashReportFingerprint(tc.message)
+
+		expLen := 2
+		if tc.ldb {
+			expLen = 1
+		}
+		if l := len(fingerprint); l != expLen {
+			t.Errorf("tc %v: Unexpected fingerprint length: %v != %v", i, l, expLen)
+		} else if msg := fingerprint[expLen-1]; msg != tc.exp {
+			t.Errorf("tc %v:\n\"%v\" !=\n\"%v\"", i, msg, tc.exp)
+		}
+	}
+}

cmd/stcrashreceiver/sourcecodeloader.go

@@ -0,0 +1,118 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	urlPrefix   = "https://raw.githubusercontent.com/syncthing/syncthing/"
+	httpTimeout = 10 * time.Second
+)
+
+type githubSourceCodeLoader struct {
+	mut     sync.Mutex
+	version string
+	cache   map[string]map[string][][]byte // version -> file -> lines
+	client  *http.Client
+}
+
+func newGithubSourceCodeLoader() *githubSourceCodeLoader {
+	return &githubSourceCodeLoader{
+		cache:  make(map[string]map[string][][]byte),
+		client: &http.Client{Timeout: httpTimeout},
+	}
+}
+
+func (l *githubSourceCodeLoader) LockWithVersion(version string) {
+	l.mut.Lock()
+	l.version = version
+	if _, ok := l.cache[version]; !ok {
+		l.cache[version] = make(map[string][][]byte)
+	}
+}
+
+func (l *githubSourceCodeLoader) Unlock() {
+	l.mut.Unlock()
+}
+
+func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]byte, int) {
+	filename = filepath.ToSlash(filename)
+	lines, ok := l.cache[l.version][filename]
+	if !ok {
+		// Cache whatever we managed to find (or nil if nothing, so we don't try again)
+		defer func() {
+			l.cache[l.version][filename] = lines
+		}()
+
+		knownPrefixes := []string{"/lib/", "/cmd/"}
+		var idx int
+		for _, pref := range knownPrefixes {
+			idx = strings.Index(filename, pref)
+			if idx >= 0 {
+				break
+			}
+		}
+		if idx == -1 {
+			return nil, 0
+		}
+
+		url := urlPrefix + l.version + filename[idx:]
+		resp, err := l.client.Get(url)
+
+		if err != nil {
+			fmt.Println("Loading source:", err)
+			return nil, 0
+		}
+		if resp.StatusCode != http.StatusOK {
+			fmt.Println("Loading source:", resp.Status)
+			return nil, 0
+		}
+		data, err := ioutil.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		if err != nil {
+			fmt.Println("Loading source:", err.Error())
+			return nil, 0
+		}
+		lines = bytes.Split(data, []byte{'\n'})
+	}
+
+	return getLineFromLines(lines, line, context)
+}
+
+func getLineFromLines(lines [][]byte, line, context int) ([][]byte, int) {
+	if lines == nil {
+		// cached error from ReadFile: return no lines
+		return nil, 0
+	}
+
+	line-- // stack trace lines are 1-indexed
+	start := line - context
+	var idx int
+	if start < 0 {
+		start = 0
+		idx = line
+	} else {
+		idx = context
+	}
+	end := line + context + 1
+	if line >= len(lines) {
+		return nil, 0
+	}
+	if end > len(lines) {
+		end = len(lines)
+	}
+	return lines[start:end], idx
+}

cmd/stcrashreceiver/stcrashreceiver.go

@@ -0,0 +1,142 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+type crashReceiver struct {
+	dir string
+	dsn string
+}
+
+func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	// The final path component should be a SHA256 hash in hex, so 64 hex
+	// characters. We don't care about case on the request but use lower
+	// case internally.
+	reportID := strings.ToLower(path.Base(req.URL.Path))
+	if len(reportID) != 64 {
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+	for _, c := range reportID {
+		if c >= 'a' && c <= 'f' {
+			continue
+		}
+		if c >= '0' && c <= '9' {
+			continue
+		}
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+
+	// The location of the report on disk, compressed
+	fullPath := filepath.Join(r.dir, r.dirFor(reportID), reportID) + ".gz"
+
+	switch req.Method {
+	case http.MethodGet:
+		r.serveGet(fullPath, w, req)
+	case http.MethodHead:
+		r.serveHead(fullPath, w, req)
+	case http.MethodPut:
+		r.servePut(reportID, fullPath, w, req)
+	default:
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+// serveGet responds to GET requests by serving the uncompressed report.
+func (r *crashReceiver) serveGet(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	fd, err := os.Open(fullPath)
+	if err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+		return
+	}
+
+	defer fd.Close()
+	gr, err := gzip.NewReader(fd)
+	if err != nil {
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+	_, _ = io.Copy(w, gr) // best effort
+}
+
+// serveHead responds to HEAD requests by checking if the named report
+// already exists in the system.
+func (r *crashReceiver) serveHead(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	if _, err := os.Lstat(fullPath); err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+	}
+}
+
+// servePut accepts and stores the given report.
+func (r *crashReceiver) servePut(reportID, fullPath string, w http.ResponseWriter, req *http.Request) {
+	// Ensure the destination directory exists
+	if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
+		log.Println("Creating directory:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Read at most maxRequestSize of report data.
+	log.Println("Receiving report", reportID)
+	lr := io.LimitReader(req.Body, maxRequestSize)
+	bs, err := ioutil.ReadAll(lr)
+	if err != nil {
+		log.Println("Reading report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Compress the report for storage
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	_, _ = gw.Write(bs) // can't fail
+	gw.Close()
+
+	// Create an output file with the compressed report
+	err = ioutil.WriteFile(fullPath, buf.Bytes(), 0644)
+	if err != nil {
+		log.Println("Saving report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Send the report to Sentry
+	if r.dsn != "" {
+		// Remote ID
+		user := userIDFor(req)
+
+		go func() {
+			// There's no need for the client to have to wait for this part.
+			pkt, err := parseCrashReport(reportID, bs)
+			if err != nil {
+				log.Println("Failed to parse crash report:", err)
+				return
+			}
+			if err := sendReport(r.dsn, pkt, user); err != nil {
+				log.Println("Failed to send crash report:", err)
+			}
+		}()
+	}
+}
+
+// 01234567890abcdef... => 01/23
+func (r *crashReceiver) dirFor(base string) string {
+	return filepath.Join(base[0:2], base[2:4])
+}

cmd/stdisco/main.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/binary"
 	"flag"
@@ -47,14 +48,16 @@ func main() {
 		log.Println("My ID:", myID)
 	}
 
-	runbeacon(beacon.NewMulticast(mc), fake)
+	ctx := context.Background()
-	runbeacon(beacon.NewBroadcast(bc), fake)
+
+	runbeacon(ctx, beacon.NewMulticast(mc), fake)
+	runbeacon(ctx, beacon.NewBroadcast(bc), fake)
 
 	select {}
 }
 
-func runbeacon(bc beacon.Interface, fake bool) {
+func runbeacon(ctx context.Context, bc beacon.Interface, fake bool) {
-	go bc.Serve()
+	go bc.Serve(ctx)
 	go recv(bc)
 	if fake {
 		go send(bc)

cmd/stdiscosrv/apisrv.go

@@ -18,6 +18,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -65,12 +66,12 @@ func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator,
 	}
 }
 
-func (s *apiSrv) Serve() {
+func (s *apiSrv) Serve(ctx context.Context) error {
 	if s.useHTTP {
 		listener, err := net.Listen("tcp", s.addr)
 		if err != nil {
 			log.Println("Listen:", err)
-			return
+			return err
 		}
 		s.listener = listener
 	} else {
@@ -92,7 +93,7 @@ func (s *apiSrv) Serve() {
 		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
 		if err != nil {
 			log.Println("Listen:", err)
-			return
+			return err
 		}
 		s.listener = tlsListener
 	}
@@ -106,9 +107,11 @@ func (s *apiSrv) Serve() {
 		MaxHeaderBytes: httpMaxHeaderBytes,
 	}
 
-	if err := srv.Serve(s.listener); err != nil {
+	err := srv.Serve(s.listener)
+	if err != nil {
 		log.Println("Serve:", err)
 	}
+	return err
 }
 
 var topCtx = context.Background()
@@ -131,11 +134,19 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 		log.Println(reqID, req.Method, req.URL)
 	}
 
-	var remoteIP net.IP
+	remoteAddr := &net.TCPAddr{
+		IP:   nil,
+		Port: -1,
+	}
+
 	if s.useHTTP {
-		remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
+		remoteAddr.IP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
+		if parsedPort, err := strconv.ParseInt(req.Header.Get("X-Client-Port"), 10, 0); err == nil {
+			remoteAddr.Port = int(parsedPort)
+		}
 	} else {
-		addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
+		var err error
+		remoteAddr, err = net.ResolveTCPAddr("tcp", req.RemoteAddr)
 		if err != nil {
 			log.Println("remoteAddr:", err)
 			lw.Header().Set("Retry-After", errorRetryAfterString())
@@ -143,14 +154,13 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 			apiRequestsTotal.WithLabelValues("no_remote_addr").Inc()
 			return
 		}
-		remoteIP = addr.IP
 	}
 
 	switch req.Method {
 	case "GET":
 		s.handleGET(ctx, lw, req)
 	case "POST":
-		s.handlePOST(ctx, remoteIP, lw, req)
+		s.handlePOST(ctx, remoteAddr, lw, req)
 	default:
 		http.Error(lw, "Method Not Allowed", http.StatusMethodNotAllowed)
 	}
@@ -216,7 +226,7 @@ func (s *apiSrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http
 	w.Write(bs)
 }
 
-func (s *apiSrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
+func (s *apiSrv) handlePOST(ctx context.Context, remoteAddr *net.TCPAddr, w http.ResponseWriter, req *http.Request) {
 	reqID := ctx.Value(idKey).(requestID)
 
 	rawCert := certificateBytes(req)
@@ -243,7 +253,7 @@ func (s *apiSrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.Respons
 
 	deviceID := protocol.NewDeviceID(rawCert)
 
-	addresses := fixupAddresses(remoteIP, ann.Addresses)
+	addresses := fixupAddresses(remoteAddr, ann.Addresses)
 	if len(addresses) == 0 {
 		announceRequestsTotal.WithLabelValues("bad_request").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
@@ -251,7 +261,7 @@ func (s *apiSrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.Respons
 		return
 	}
 
-	if err := s.handleAnnounce(remoteIP, deviceID, addresses); err != nil {
+	if err := s.handleAnnounce(deviceID, addresses); err != nil {
 		announceRequestsTotal.WithLabelValues("internal_error").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
@@ -268,7 +278,7 @@ func (s *apiSrv) Stop() {
 	s.listener.Close()
 }
 
-func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addresses []string) error {
+func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string) error {
 	key := deviceID.String()
 	now := time.Now()
 	expire := now.Add(addressExpiryTime).UnixNano()
@@ -279,6 +289,10 @@ func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addre
 		dbAddrs[i].Expires = expire
 	}
 
+	// The address slice must always be sorted for database merges to work
+	// properly.
+	sort.Sort(databaseAddressOrder(dbAddrs))
+
 	seen := now.UnixNano()
 	if s.repl != nil {
 		s.repl.send(key, dbAddrs, seen)
@@ -295,33 +309,71 @@ func certificateBytes(req *http.Request) []byte {
 		return req.TLS.PeerCertificates[0].Raw
 	}
 
+	var bs []byte
+
 	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
-		bs := []byte(hdr)
+		if strings.Contains(hdr, "%") {
-		// The certificate is in PEM format but with spaces for newlines. We
+			// Nginx using $ssl_client_escaped_cert
-		// need to reinstate the newlines for the PEM decoder. But we need to
+			// The certificate is in PEM format with url encoding.
-		// leave the spaces in the BEGIN and END lines - the first and last
+			// We need to decode for the PEM decoder
-		// space - alone.
+			hdr, err := url.QueryUnescape(hdr)
-		firstSpace := bytes.Index(bs, []byte(" "))
+			if err != nil {
-		lastSpace := bytes.LastIndex(bs, []byte(" "))
+				// Decoding failed
-		for i := firstSpace + 1; i < lastSpace; i++ {
+				return nil
-			if bs[i] == ' ' {
+			}
-				bs[i] = '\n'
+
+			bs = []byte(hdr)
+		} else {
+			// Nginx using $ssl_client_cert
+			// The certificate is in PEM format but with spaces for newlines. We
+			// need to reinstate the newlines for the PEM decoder. But we need to
+			// leave the spaces in the BEGIN and END lines - the first and last
+			// space - alone.
+			bs = []byte(hdr)
+			firstSpace := bytes.Index(bs, []byte(" "))
+			lastSpace := bytes.LastIndex(bs, []byte(" "))
+			for i := firstSpace + 1; i < lastSpace; i++ {
+				if bs[i] == ' ' {
+					bs[i] = '\n'
+				}
 			}
 		}
-		block, _ := pem.Decode(bs)
+	} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
-		if block == nil {
+		// Traefik 2 passtlsclientcert
+		// The certificate is in PEM format with url encoding but without newlines
+		// and start/end statements. We need to decode, reinstate the newlines every 64
+		// character and add statements for the PEM decoder
+		hdr, err := url.QueryUnescape(hdr)
+		if err != nil {
 			// Decoding failed
 			return nil
 		}
-		return block.Bytes
+
+		for i := 64; i < len(hdr); i += 65 {
+			hdr = hdr[:i] + "\n" + hdr[i:]
+		}
+
+		hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
+		hdr = hdr + "\n-----END CERTIFICATE-----\n"
+		bs = []byte(hdr)
 	}
 
-	return nil
+	if bs == nil {
+		return nil
+	}
+
+	block, _ := pem.Decode(bs)
+	if block == nil {
+		// Decoding failed
+		return nil
+	}
+
+	return block.Bytes
 }
 
 // fixupAddresses checks the list of addresses, removing invalid ones and
 // replacing unspecified IPs with the given remote IP.
-func fixupAddresses(remote net.IP, addresses []string) []string {
+func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
 	fixed := make([]string, 0, len(addresses))
 	for _, annAddr := range addresses {
 		uri, err := url.Parse(annAddr)
@@ -341,27 +393,34 @@ func fixupAddresses(remote net.IP, addresses []string) []string {
 			continue
 		}
 
-		if host == "" || ip.IsUnspecified() {
+		if remote != nil {
-			// Replace the unspecified IP with the request source.
+			if host == "" || ip.IsUnspecified() {
+				// Replace the unspecified IP with the request source.
 
-			// ... unless the request source is the loopback address or
+				// ... unless the request source is the loopback address or
-			// multicast/unspecified (can't happen, really).
+				// multicast/unspecified (can't happen, really).
-			if remote.IsLoopback() || remote.IsMulticast() || remote.IsUnspecified() {
+				if remote.IP.IsLoopback() || remote.IP.IsMulticast() || remote.IP.IsUnspecified() {
-				continue
+					continue
+				}
+
+				// Do not use IPv6 remote address if requested scheme is ...4
+				// (i.e., tcp4, etc.)
+				if strings.HasSuffix(uri.Scheme, "4") && remote.IP.To4() == nil {
+					continue
+				}
+
+				// Do not use IPv4 remote address if requested scheme is ...6
+				if strings.HasSuffix(uri.Scheme, "6") && remote.IP.To4() != nil {
+					continue
+				}
+
+				host = remote.IP.String()
 			}
 
-			// Do not use IPv6 remote address if requested scheme is ...4
+			// If zero port was specified, use remote port.
-			// (i.e., tcp4, etc.)
+			if port == "0" && remote.Port > 0 {
-			if strings.HasSuffix(uri.Scheme, "4") && remote.To4() == nil {
+				port = fmt.Sprintf("%d", remote.Port)
-				continue
 			}
-
-			// Do not use IPv4 remote address if requested scheme is ...6
-			if strings.HasSuffix(uri.Scheme, "6") && remote.To4() != nil {
-				continue
-			}
-
-			host = remote.String()
 		}
 
 		uri.Host = net.JoinHostPort(host, port)

cmd/stdiscosrv/apisrv_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestFixupAddresses(t *testing.T) {
 	cases := []struct {
-		remote net.IP
+		remote *net.TCPAddr
 		in     []string
 		out    []string
 	}{
@@ -22,37 +22,53 @@ func TestFixupAddresses(t *testing.T) {
 			in:  []string{"tcp://1.2.3.4:22000"},
 			out: []string{"tcp://1.2.3.4:22000"},
 		}, { // unspecified replaced by remote
-			remote: net.ParseIP("1.2.3.4"),
+			remote: addr("1.2.3.4", 22000),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://1.2.3.4:22000", "tcp://192.0.2.42:22000"},
 		}, { // unspecified not used as replacement
-			remote: net.ParseIP("0.0.0.0"),
+			remote: addr("0.0.0.0", 22000),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // unspecified not used as replacement
-			remote: net.ParseIP("::"),
+			remote: addr("::", 22000),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // localhost not used as replacement
-			remote: net.ParseIP("127.0.0.1"),
+			remote: addr("127.0.0.1", 22000),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // localhost not used as replacement
-			remote: net.ParseIP("::1"),
+			remote: addr("::1", 22000),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // multicast not used as replacement
-			remote: net.ParseIP("224.0.0.1"),
+			remote: addr("224.0.0.1", 22000),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // multicast not used as replacement
-			remote: net.ParseIP("ff80::42"),
+			remote: addr("ff80::42", 22000),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // explicitly announced weirdness is also filtered
-			remote: net.ParseIP("192.0.2.42"),
+			remote: addr("192.0.2.42", 22000),
 			in:     []string{"tcp://:22000", "tcp://127.1.2.3:22000", "tcp://[::1]:22000", "tcp://[ff80::42]:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // port remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://0.0.0.0:0"},
+			out:    []string{"tcp://123.123.123.123:9000"},
+		}, { // unspecified port remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://:0"},
+			out:    []string{"tcp://123.123.123.123:9000"},
+		}, { // empty remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://"},
+			out:    []string{},
+		}, { // port only remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://44.44.44.44:0"},
+			out:    []string{"tcp://44.44.44.44:9000"},
 		},
 	}
 
@@ -63,3 +79,10 @@ func TestFixupAddresses(t *testing.T) {
 		}
 	}
 }
+
+func addr(host string, port int) *net.TCPAddr {
+	return &net.TCPAddr{
+		IP:   net.ParseIP(host),
+		Port: port,
+	}
+}

cmd/stdiscosrv/database.go

@@ -4,12 +4,14 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//go:generate go run ../../script/protofmt.go database.proto
+//go:generate go run ../../proto/scripts/protofmt.go database.proto
 //go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
 
 package main
 
 import (
+	"context"
+	"log"
 	"sort"
 	"time"
 
@@ -36,7 +38,6 @@ type database interface {
 type levelDBStore struct {
 	db         *leveldb.DB
 	inbox      chan func()
-	stop       chan struct{}
 	clock      clock
 	marshalBuf []byte
 }
@@ -49,7 +50,6 @@ func newLevelDBStore(dir string) (*levelDBStore, error) {
 	return &levelDBStore{
 		db:    db,
 		inbox: make(chan func(), 16),
-		stop:  make(chan struct{}),
 		clock: defaultClock{},
 	}, nil
 }
@@ -154,7 +154,7 @@ func (s *levelDBStore) get(key string) (DatabaseRecord, error) {
 	return rec, nil
 }
 
-func (s *levelDBStore) Serve() {
+func (s *levelDBStore) Serve(ctx context.Context) error {
 	t := time.NewTimer(0)
 	defer t.Stop()
 	defer s.db.Close()
@@ -182,7 +182,7 @@ loop:
 			// the next.
 			t.Reset(databaseStatisticsInterval)
 
-		case <-s.stop:
+		case <-ctx.Done():
 			// We're done.
 			close(statisticsTrigger)
 			break loop
@@ -191,6 +191,8 @@ loop:
 
 	// Also wait for statisticsServe to return
 	<-statisticsDone
+
+	return nil
 }
 
 func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
@@ -254,21 +256,20 @@ func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- stru
 	}
 }
 
-func (s *levelDBStore) Stop() {
-	close(s.stop)
-}
-
 // merge returns the merged result of the two database records a and b. The
 // result is the union of the two address sets, with the newer expiry time
 // chosen for any duplicates.
 func merge(a, b DatabaseRecord) DatabaseRecord {
 	// Both lists must be sorted for this to work.
-	sort.Slice(a.Addresses, func(i, j int) bool {
+	if !sort.IsSorted(databaseAddressOrder(a.Addresses)) {
-		return a.Addresses[i].Address < a.Addresses[j].Address
+		log.Println("Warning: bug: addresses not correctly sorted in merge")
-	})
+		a.Addresses = sortedAddressCopy(a.Addresses)
-	sort.Slice(b.Addresses, func(i, j int) bool {
+	}
-		return b.Addresses[i].Address < b.Addresses[j].Address
+	if !sort.IsSorted(databaseAddressOrder(b.Addresses)) {
-	})
+		// no warning because this is the side we read from disk and it may
+		// legitimately predate correct sorting.
+		b.Addresses = sortedAddressCopy(b.Addresses)
+	}
 
 	res := DatabaseRecord{
 		Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
@@ -352,3 +353,24 @@ func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
 	}
 	return addrs
 }
+
+func sortedAddressCopy(addrs []DatabaseAddress) []DatabaseAddress {
+	sorted := make([]DatabaseAddress, len(addrs))
+	copy(sorted, addrs)
+	sort.Sort(databaseAddressOrder(sorted))
+	return sorted
+}
+
+type databaseAddressOrder []DatabaseAddress
+
+func (s databaseAddressOrder) Less(a, b int) bool {
+	return s[a].Address < s[b].Address
+}
+
+func (s databaseAddressOrder) Swap(a, b int) {
+	s[a], s[b] = s[b], s[a]
+}
+
+func (s databaseAddressOrder) Len() int {
+	return len(s)
+}

cmd/stdiscosrv/database.pb.go

@@ -3,12 +3,14 @@
 
 package main
 
-import proto "github.com/gogo/protobuf/proto"
+import (
-import fmt "fmt"
+	fmt "fmt"
-import math "math"
+	_ "github.com/gogo/protobuf/gogoproto"
-import _ "github.com/gogo/protobuf/gogoproto"
+	proto "github.com/gogo/protobuf/proto"
-
+	io "io"
-import io "io"
+	math "math"
+	math_bits "math/bits"
+)
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
@@ -19,7 +21,7 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
 
 type DatabaseRecord struct {
 	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
@@ -32,7 +34,7 @@ func (m *DatabaseRecord) Reset()         { *m = DatabaseRecord{} }
 func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
 func (*DatabaseRecord) ProtoMessage()    {}
 func (*DatabaseRecord) Descriptor() ([]byte, []int) {
-	return fileDescriptor_database_0f49e029703a04f5, []int{0}
+	return fileDescriptor_b90fe3356ea5df07, []int{0}
 }
 func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -42,15 +44,15 @@ func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, erro
 		return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalTo(b)
+		n, err := m.MarshalToSizedBuffer(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (dst *DatabaseRecord) XXX_Merge(src proto.Message) {
+func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DatabaseRecord.Merge(dst, src)
+	xxx_messageInfo_DatabaseRecord.Merge(m, src)
 }
 func (m *DatabaseRecord) XXX_Size() int {
 	return m.Size()
@@ -71,7 +73,7 @@ func (m *ReplicationRecord) Reset()         { *m = ReplicationRecord{} }
 func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
 func (*ReplicationRecord) ProtoMessage()    {}
 func (*ReplicationRecord) Descriptor() ([]byte, []int) {
-	return fileDescriptor_database_0f49e029703a04f5, []int{1}
+	return fileDescriptor_b90fe3356ea5df07, []int{1}
 }
 func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -81,15 +83,15 @@ func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, e
 		return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalTo(b)
+		n, err := m.MarshalToSizedBuffer(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (dst *ReplicationRecord) XXX_Merge(src proto.Message) {
+func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ReplicationRecord.Merge(dst, src)
+	xxx_messageInfo_ReplicationRecord.Merge(m, src)
 }
 func (m *ReplicationRecord) XXX_Size() int {
 	return m.Size()
@@ -109,7 +111,7 @@ func (m *DatabaseAddress) Reset()         { *m = DatabaseAddress{} }
 func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
 func (*DatabaseAddress) ProtoMessage()    {}
 func (*DatabaseAddress) Descriptor() ([]byte, []int) {
-	return fileDescriptor_database_0f49e029703a04f5, []int{2}
+	return fileDescriptor_b90fe3356ea5df07, []int{2}
 }
 func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -119,15 +121,15 @@ func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, err
 		return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalTo(b)
+		n, err := m.MarshalToSizedBuffer(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (dst *DatabaseAddress) XXX_Merge(src proto.Message) {
+func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DatabaseAddress.Merge(dst, src)
+	xxx_messageInfo_DatabaseAddress.Merge(m, src)
 }
 func (m *DatabaseAddress) XXX_Size() int {
 	return m.Size()
@@ -143,10 +145,34 @@ func init() {
 	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
 	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
 }
+
+func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
+
+var fileDescriptor_b90fe3356ea5df07 = []byte{
+	// 270 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
+	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
+	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
+	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
+	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
+	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
+	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
+	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
+	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
+	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
+	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
+	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
+	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
+	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
+	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
+	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
+	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
+}
+
 func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -154,44 +180,51 @@ func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Addresses) > 0 {
+	if m.Missed != 0 {
-		for _, msg := range m.Addresses {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
-			dAtA[i] = 0xa
+		i--
-			i++
+		dAtA[i] = 0x20
-			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
-			n, err := msg.MarshalTo(dAtA[i:])
-			if err != nil {
-				return 0, err
-			}
-			i += n
-		}
-	}
-	if m.Misses != 0 {
-		dAtA[i] = 0x10
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
 	}
 	if m.Seen != 0 {
-		dAtA[i] = 0x18
-		i++
 		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
 	}
-	if m.Missed != 0 {
+	if m.Misses != 0 {
-		dAtA[i] = 0x20
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
-		i++
+		i--
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+		dAtA[i] = 0x10
 	}
-	return i, nil
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
 }
 
 func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -199,40 +232,48 @@ func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Key) > 0 {
+	if m.Seen != 0 {
-		dAtA[i] = 0xa
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
-		i++
+		i--
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		dAtA[i] = 0x18
-		i += copy(dAtA[i:], m.Key)
 	}
 	if len(m.Addresses) > 0 {
-		for _, msg := range m.Addresses {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
-			dAtA[i] = 0x12
+			{
-			i++
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
+				if err != nil {
-			n, err := msg.MarshalTo(dAtA[i:])
+					return 0, err
-			if err != nil {
+				}
-				return 0, err
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
 			}
-			i += n
+			i--
+			dAtA[i] = 0x12
 		}
 	}
-	if m.Seen != 0 {
+	if len(m.Key) > 0 {
-		dAtA[i] = 0x18
+		i -= len(m.Key)
-		i++
+		copy(dAtA[i:], m.Key)
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		i--
+		dAtA[i] = 0xa
 	}
-	return i, nil
+	return len(dAtA) - i, nil
 }
 
 func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -240,32 +281,40 @@ func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Address) > 0 {
-		dAtA[i] = 0xa
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
-		i += copy(dAtA[i:], m.Address)
-	}
 	if m.Expires != 0 {
-		dAtA[i] = 0x10
-		i++
 		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
+		i--
+		dAtA[i] = 0x10
 	}
-	return i, nil
+	if len(m.Address) > 0 {
+		i -= len(m.Address)
+		copy(dAtA[i:], m.Address)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
 }
 
 func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
+	offset -= sovDatabase(v)
+	base := offset
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
 		v >>= 7
 		offset++
 	}
 	dAtA[offset] = uint8(v)
-	return offset + 1
+	return base
 }
 func (m *DatabaseRecord) Size() (n int) {
 	if m == nil {
@@ -330,14 +379,7 @@ func (m *DatabaseAddress) Size() (n int) {
 }
 
 func sovDatabase(x uint64) (n int) {
-	for {
+	return (math_bits.Len64(x|1) + 6) / 7
-		n++
-		x >>= 7
-		if x == 0 {
-			break
-		}
-	}
-	return n
 }
 func sozDatabase(x uint64) (n int) {
 	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
@@ -357,7 +399,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -385,7 +427,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= (int(b) & 0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -394,6 +436,9 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -416,7 +461,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Misses |= (int32(b) & 0x7F) << shift
+				m.Misses |= int32(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -435,7 +480,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= (int64(b) & 0x7F) << shift
+				m.Seen |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -454,7 +499,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Missed |= (int64(b) & 0x7F) << shift
+				m.Missed |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -468,6 +513,9 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -495,7 +543,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -523,7 +571,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -533,6 +581,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -552,7 +603,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= (int(b) & 0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -561,6 +612,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -583,7 +637,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= (int64(b) & 0x7F) << shift
+				m.Seen |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -597,6 +651,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -624,7 +681,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -652,7 +709,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -662,6 +719,9 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -681,7 +741,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Expires |= (int64(b) & 0x7F) << shift
+				m.Expires |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -695,6 +755,9 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -710,6 +773,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 func skipDatabase(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0
+	depth := 0
 	for iNdEx < l {
 		var wire uint64
 		for shift := uint(0); ; shift += 7 {
@@ -741,10 +805,8 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			return iNdEx, nil
 		case 1:
 			iNdEx += 8
-			return iNdEx, nil
 		case 2:
 			var length int
 			for shift := uint(0); ; shift += 7 {
@@ -761,76 +823,34 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			iNdEx += length
 			if length < 0 {
 				return 0, ErrInvalidLengthDatabase
 			}
-			return iNdEx, nil
+			iNdEx += length
 		case 3:
-			for {
+			depth++
-				var innerWire uint64
-				var start int = iNdEx
-				for shift := uint(0); ; shift += 7 {
-					if shift >= 64 {
-						return 0, ErrIntOverflowDatabase
-					}
-					if iNdEx >= l {
-						return 0, io.ErrUnexpectedEOF
-					}
-					b := dAtA[iNdEx]
-					iNdEx++
-					innerWire |= (uint64(b) & 0x7F) << shift
-					if b < 0x80 {
-						break
-					}
-				}
-				innerWireType := int(innerWire & 0x7)
-				if innerWireType == 4 {
-					break
-				}
-				next, err := skipDatabase(dAtA[start:])
-				if err != nil {
-					return 0, err
-				}
-				iNdEx = start + next
-			}
-			return iNdEx, nil
 		case 4:
-			return iNdEx, nil
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupDatabase
+			}
+			depth--
 		case 5:
 			iNdEx += 4
-			return iNdEx, nil
 		default:
 			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
 		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthDatabase
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
 	}
-	panic("unreachable")
+	return 0, io.ErrUnexpectedEOF
 }
 
 var (
-	ErrInvalidLengthDatabase = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrInvalidLengthDatabase        = fmt.Errorf("proto: negative length found during unmarshaling")
-	ErrIntOverflowDatabase   = fmt.Errorf("proto: integer overflow")
+	ErrIntOverflowDatabase          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupDatabase = fmt.Errorf("proto: unexpected end of group")
 )
-
-func init() { proto.RegisterFile("database.proto", fileDescriptor_database_0f49e029703a04f5) }
-
-var fileDescriptor_database_0f49e029703a04f5 = []byte{
-	// 270 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
-	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
-	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
-	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
-	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
-	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
-	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
-	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
-	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
-	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
-	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
-	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
-	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
-	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
-	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
-	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
-	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
-}

cmd/stdiscosrv/database_test.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"testing"
@@ -20,8 +21,9 @@ func TestDatabaseGetSet(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	go db.Serve()
+	ctx, cancel := context.WithCancel(context.Background())
-	defer db.Stop()
+	go db.Serve(ctx)
+	defer cancel()
 
 	// Check missing record
 

cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv

@@ -0,0 +1,4 @@
+[stdiscosrv]
+title=Syncthing discovery server
+description=Lets syncthing clients discover each other
+ports=8443/tcp

cmd/stdiscosrv/etc/linux-systemd/default

@@ -0,0 +1,3 @@
+# Default settings for syncthing-discosrv (stdiscosrv).
+## Add Options here:
+DISCOSRV_OPTS=

cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=Syncthing Discovery Server
+After=network.target
+Documentation=man:stdiscosrv(1)
+
+[Service]
+WorkingDirectory=/var/lib/syncthing-discosrv
+EnvironmentFile=/etc/default/syncthing-discosrv
+ExecStart=/usr/bin/stdiscosrv $DISCOSRV_OPTS
+
+# Hardening
+User=syncthing-discosrv
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-discosrv
+NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
+
+[Install]
+WantedBy=multi-user.target
+Alias=syncthing-discosrv.service

cmd/stdiscosrv/main.go

@@ -7,23 +7,22 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
-	"fmt"
 	"log"
 	"net"
 	"net/http"
 	"os"
-	"runtime"
-	"strconv"
 	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/thejerf/suture"
+	"github.com/thejerf/suture/v4"
 )
 
 const (
@@ -65,24 +64,6 @@ var levelDBOptions = &opt.Options{
 	WriteBuffer: 32 << 20, // default 4<<20
 }
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`stdiscosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
-}
-
 var (
 	debug = false
 )
@@ -109,19 +90,24 @@ func main() {
 	flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
 	flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
 	flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
-	log.Println(LongVersion)
+	log.Println(build.LongVersionFor("stdiscosrv"))
+	if *showVersion {
+		return
+	}
 
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
-	if err != nil {
+	if os.IsNotExist(err) {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
+	} else if err != nil {
+		log.Fatalln("Failed to load keypair:", err)
 	}
-
 	devID := protocol.NewDeviceID(cert.Certificate[0])
 	log.Println("Server device ID is", devID)
 
@@ -198,5 +184,5 @@ func main() {
 	}
 
 	// Engage!
-	main.Serve()
+	main.Serve(context.Background())
 }

cmd/stdiscosrv/replication.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"encoding/binary"
 	"fmt"
@@ -32,7 +33,6 @@ type replicationSender struct {
 	cert       tls.Certificate // our certificate
 	allowedIDs []protocol.DeviceID
 	outbox     chan ReplicationRecord
-	stop       chan struct{}
 }
 
 func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
@@ -41,11 +41,10 @@ func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protoco
 		cert:       cert,
 		allowedIDs: allowedIDs,
 		outbox:     make(chan ReplicationRecord, replicationOutboxSize),
-		stop:       make(chan struct{}),
 	}
 }
 
-func (s *replicationSender) Serve() {
+func (s *replicationSender) Serve(ctx context.Context) error {
 	// Sleep a little at startup. Peers often restart at the same time, and
 	// this avoid the service failing and entering backoff state
 	// unnecessarily, while also reducing the reconnect rate to something
@@ -62,7 +61,7 @@ func (s *replicationSender) Serve() {
 	conn, err := tls.Dial("tcp", s.dst, tlsCfg)
 	if err != nil {
 		log.Println("Replication connect:", err)
-		return
+		return err
 	}
 	defer func() {
 		conn.SetWriteDeadline(time.Now().Add(time.Second))
@@ -73,13 +72,13 @@ func (s *replicationSender) Serve() {
 	remoteID, err := deviceID(conn)
 	if err != nil {
 		log.Println("Replication connect:", err)
-		return
+		return err
 	}
 
 	// Verify it's in the set of allowed device IDs.
 	if !deviceIDIn(remoteID, s.allowedIDs) {
 		log.Println("Replication connect: unexpected device ID:", remoteID)
-		return
+		return err
 	}
 
 	heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
@@ -122,20 +121,16 @@ func (s *replicationSender) Serve() {
 				replicationSendsTotal.WithLabelValues("error").Inc()
 				log.Println("Replication write:", err)
 				// Yes, we are loosing the replication event here.
-				return
+				return err
 			}
 			replicationSendsTotal.WithLabelValues("success").Inc()
 
-		case <-s.stop:
+		case <-ctx.Done():
-			return
+			return nil
 		}
 	}
 }
 
-func (s *replicationSender) Stop() {
-	close(s.stop)
-}
-
 func (s *replicationSender) String() string {
 	return fmt.Sprintf("replicationSender(%q)", s.dst)
 }
@@ -172,7 +167,6 @@ type replicationListener struct {
 	cert       tls.Certificate
 	allowedIDs []protocol.DeviceID
 	db         database
-	stop       chan struct{}
 }
 
 func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
@@ -181,11 +175,10 @@ func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []prot
 		cert:       cert,
 		allowedIDs: allowedIDs,
 		db:         db,
-		stop:       make(chan struct{}),
 	}
 }
 
-func (l *replicationListener) Serve() {
+func (l *replicationListener) Serve(ctx context.Context) error {
 	tlsCfg := &tls.Config{
 		Certificates:       []tls.Certificate{l.cert},
 		ClientAuth:         tls.RequestClientCert,
@@ -196,14 +189,14 @@ func (l *replicationListener) Serve() {
 	lst, err := tls.Listen("tcp", l.addr, tlsCfg)
 	if err != nil {
 		log.Println("Replication listen:", err)
-		return
+		return err
 	}
 	defer lst.Close()
 
 	for {
 		select {
-		case <-l.stop:
+		case <-ctx.Done():
-			return
+			return nil
 		default:
 		}
 
@@ -211,7 +204,7 @@ func (l *replicationListener) Serve() {
 		conn, err := lst.Accept()
 		if err != nil {
 			log.Println("Replication accept:", err)
-			return
+			return err
 		}
 
 		// Figure out the other side device ID
@@ -231,19 +224,15 @@ func (l *replicationListener) Serve() {
 			continue
 		}
 
-		go l.handle(conn)
+		go l.handle(ctx, conn)
 	}
 }
 
-func (l *replicationListener) Stop() {
-	close(l.stop)
-}
-
 func (l *replicationListener) String() string {
 	return fmt.Sprintf("replicationListener(%q)", l.addr)
 }
 
-func (l *replicationListener) handle(conn net.Conn) {
+func (l *replicationListener) handle(ctx context.Context, conn net.Conn) {
 	defer func() {
 		conn.SetWriteDeadline(time.Now().Add(time.Second))
 		conn.Close()
@@ -253,7 +242,7 @@ func (l *replicationListener) handle(conn net.Conn) {
 
 	for {
 		select {
-		case <-l.stop:
+		case <-ctx.Done():
 			return
 		default:
 		}

cmd/stdiscosrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-discosrv --ingroup syncthing syncthing-discosrv

cmd/stevents/main.go

@@ -28,16 +28,21 @@ func main() {
 	log.SetFlags(0)
 
 	target := flag.String("target", "localhost:8384", "Target Syncthing instance")
+	types := flag.String("types", "", "Filter for specific event types (comma-separated)")
 	apikey := flag.String("apikey", "", "Syncthing API key")
 	flag.Parse()
 
 	if *apikey == "" {
 		log.Fatal("Must give -apikey argument")
 	}
+	var eventsArg string
+	if len(*types) > 0 {
+		eventsArg = "&events=" + *types
+	}
 
 	since := 0
 	for {
-		req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d", *target, since), nil)
+		req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d%s", *target, since, eventsArg), nil)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/stfinddevice/main.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"errors"
 	"flag"
@@ -17,6 +18,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -82,7 +84,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 }
 
 func checkServer(deviceID protocol.DeviceID, server string) checkResult {
-	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger)
 	if err != nil {
 		return checkResult{error: err}
 	}
@@ -94,7 +96,7 @@ func checkServer(deviceID protocol.DeviceID, server string) checkResult {
 	})
 
 	go func() {
-		addresses, err := disco.Lookup(deviceID)
+		addresses, err := disco.Lookup(context.Background(), deviceID)
 		res <- checkResult{addresses: addresses, error: err}
 	}()
 

cmd/stindex/accounting.go

@@ -0,0 +1,58 @@
+// Copyright (C) 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"text/tabwriter"
+
+	"github.com/syncthing/syncthing/lib/db/backend"
+)
+
+// account prints key and data size statistics per class
+func account(ldb backend.Backend) {
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var ksizes [256]int
+	var dsizes [256]int
+	var counts [256]int
+	var max [256]int
+
+	for it.Next() {
+		key := it.Key()
+		t := key[0]
+		ds := len(it.Value())
+		ks := len(key)
+		s := ks + ds
+
+		counts[t]++
+		ksizes[t] += ks
+		dsizes[t] += ds
+		if s > max[t] {
+			max[t] = s
+		}
+	}
+
+	tw := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', tabwriter.AlignRight)
+	toti, totds, totks := 0, 0, 0
+	for t := range ksizes {
+		if ksizes[t] > 0 {
+			// yes metric kilobytes 🤘
+			fmt.Fprintf(tw, "0x%02x:\t%d items,\t%d KB keys +\t%d KB data,\t%d B +\t%d B avg,\t%d B max\t\n", t, counts[t], ksizes[t]/1000, dsizes[t]/1000, ksizes[t]/counts[t], dsizes[t]/counts[t], max[t])
+			toti += counts[t]
+			totds += dsizes[t]
+			totks += ksizes[t]
+		}
+	}
+	fmt.Fprintf(tw, "Total\t%d items,\t%d KB keys +\t%d KB data.\t\n", toti, totks/1000, totds/1000)
+	tw.Flush()
+}

cmd/stindex/dump.go

@@ -13,11 +13,15 @@ import (
 	"time"
 
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func dump(ldb *db.Lowlevel) {
+func dump(ldb backend.Backend) {
-	it := ldb.NewIterator(nil, nil)
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {
@@ -48,10 +52,10 @@ func dump(ldb *db.Lowlevel) {
 			fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
 
 		case db.KeyTypeDeviceStatistic:
-			fmt.Printf("[dstat] K:%x V:%x\n", it.Key(), it.Value())
+			fmt.Printf("[dstat] K:%x V:%x\n", key, it.Value())
 
 		case db.KeyTypeFolderStatistic:
-			fmt.Printf("[fstat] K:%x V:%x\n", it.Key(), it.Value())
+			fmt.Printf("[fstat] K:%x V:%x\n", key, it.Value())
 
 		case db.KeyTypeVirtualMtime:
 			folder := binary.BigEndian.Uint32(key[1:])
@@ -63,21 +67,89 @@ func dump(ldb *db.Lowlevel) {
 			fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, real, virt)
 
 		case db.KeyTypeFolderIdx:
-			key := binary.BigEndian.Uint32(it.Key()[1:])
+			key := binary.BigEndian.Uint32(key[1:])
 			fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
 
 		case db.KeyTypeDeviceIdx:
-			key := binary.BigEndian.Uint32(it.Key()[1:])
+			key := binary.BigEndian.Uint32(key[1:])
 			val := it.Value()
-			if len(val) == 0 {
+			device := "<nil>"
-				fmt.Printf("[deviceidx] K:%d V:<nil>\n", key)
+			if len(val) > 0 {
+				dev, err := protocol.DeviceIDFromBytes(val)
+				if err != nil {
+					device = fmt.Sprintf("<invalid %d bytes>", len(val))
+				} else {
+					device = dev.String()
+				}
+			}
+			fmt.Printf("[deviceidx] K:%d V:%s\n", key, device)
+
+		case db.KeyTypeIndexID:
+			device := binary.BigEndian.Uint32(key[1:])
+			folder := binary.BigEndian.Uint32(key[5:])
+			fmt.Printf("[indexid] D:%d F:%d I:%x\n", device, folder, it.Value())
+
+		case db.KeyTypeFolderMeta:
+			folder := binary.BigEndian.Uint32(key[1:])
+			fmt.Printf("[foldermeta] F:%d", folder)
+			var cs db.CountsSet
+			if err := cs.Unmarshal(it.Value()); err != nil {
+				fmt.Printf(" (invalid)\n")
 			} else {
-				dev := protocol.DeviceIDFromBytes(val)
+				fmt.Printf(" V:%v\n", cs)
-				fmt.Printf("[deviceidx] K:%d V:%s\n", key, dev)
 			}
 
+		case db.KeyTypeMiscData:
+			fmt.Printf("[miscdata] K:%q V:%q\n", key[1:], it.Value())
+
+		case db.KeyTypeSequence:
+			folder := binary.BigEndian.Uint32(key[1:])
+			seq := binary.BigEndian.Uint64(key[5:])
+			fmt.Printf("[sequence] F:%d S:%d V:%q\n", folder, seq, it.Value())
+
+		case db.KeyTypeNeed:
+			folder := binary.BigEndian.Uint32(key[1:])
+			file := string(key[5:])
+			fmt.Printf("[need] F:%d V:%q\n", folder, file)
+
+		case db.KeyTypeBlockList:
+			fmt.Printf("[blocklist] H:%x\n", key[1:])
+
+		case db.KeyTypeBlockListMap:
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[5:37]
+			fileName := string(key[37:])
+			fmt.Printf("[blocklistmap] F:%d H:%x N:%s\n", folder, hash, fileName)
+
+		case db.KeyTypeVersion:
+			fmt.Printf("[version] H:%x", key[1:])
+			var v protocol.Vector
+			err := v.Unmarshal(it.Value())
+			if err != nil {
+				fmt.Printf(" (invalid)\n")
+			} else {
+				fmt.Printf(" V:%v\n", v)
+			}
+
+		case db.KeyTypePendingFolder:
+			device := binary.BigEndian.Uint32(key[1:])
+			folder := string(key[5:])
+			var of db.ObservedFolder
+			of.Unmarshal(it.Value())
+			fmt.Printf("[pendingFolder] D:%d F:%s V:%v\n", device, folder, of)
+
+		case db.KeyTypePendingDevice:
+			device := "<invalid>"
+			dev, err := protocol.DeviceIDFromBytes(key[1:])
+			if err == nil {
+				device = dev.String()
+			}
+			var od db.ObservedDevice
+			od.Unmarshal(it.Value())
+			fmt.Printf("[pendingDevice] D:%v V:%v\n", device, od)
+
 		default:
-			fmt.Printf("[???]\n  %x\n  %x\n", it.Key(), it.Value())
+			fmt.Printf("[??? %d]\n  %x\n  %x\n", key[0], key, it.Value())
 		}
 	}
 }

cmd/stindex/dumpsize.go

@@ -10,8 +10,10 @@ import (
 	"container/heap"
 	"encoding/binary"
 	"fmt"
+	"log"
 
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
 )
 
 type SizedElement struct {
@@ -37,11 +39,14 @@ func (h *ElementHeap) Pop() interface{} {
 	return x
 }
 
-func dumpsize(ldb *db.Lowlevel) {
+func dumpsize(ldb backend.Backend) {
 	h := &ElementHeap{}
 	heap.Init(h)
 
-	it := ldb.NewIterator(nil, nil)
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
 	var ele SizedElement
 	for it.Next() {
 		key := it.Key()

cmd/stindex/idxck.go

@@ -10,8 +10,11 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
+	"log"
+	"sort"
 
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -31,7 +34,7 @@ type sequenceKey struct {
 	sequence uint64
 }
 
-func idxck(ldb *db.Lowlevel) (success bool) {
+func idxck(ldb backend.Backend) (success bool) {
 	folders := make(map[uint32]string)
 	devices := make(map[uint32]string)
 	deviceToIDs := make(map[string]uint32)
@@ -39,10 +42,17 @@ func idxck(ldb *db.Lowlevel) (success bool) {
 	globals := make(map[globalKey]db.VersionList)
 	sequences := make(map[sequenceKey]string)
 	needs := make(map[globalKey]struct{})
+	blocklists := make(map[string]struct{})
+	versions := make(map[string]protocol.Vector)
+	usedBlocklists := make(map[string]struct{})
+	usedVersions := make(map[string]struct{})
 	var localDeviceKey uint32
 	success = true
 
-	it := ldb.NewIterator(nil, nil)
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		log.Fatal(err)
+	}
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {
@@ -94,6 +104,20 @@ func idxck(ldb *db.Lowlevel) (success bool) {
 			folder := binary.BigEndian.Uint32(key[1:])
 			name := nulString(key[1+4:])
 			needs[globalKey{folder, name}] = struct{}{}
+
+		case db.KeyTypeBlockList:
+			hash := string(key[1:])
+			blocklists[hash] = struct{}{}
+
+		case db.KeyTypeVersion:
+			hash := string(key[1:])
+			var v protocol.Vector
+			if err := v.Unmarshal(it.Value()); err != nil {
+				fmt.Println("Unable to unmarshal Vector:", err)
+				success = false
+				continue
+			}
+			versions[hash] = v
 		}
 	}
 
@@ -103,6 +127,7 @@ func idxck(ldb *db.Lowlevel) (success bool) {
 		return
 	}
 
+	var missingSeq []sequenceKey
 	for fk, fi := range fileInfos {
 		if fk.name != fi.Name {
 			fmt.Printf("Mismatching FileInfo name, %q (key) != %q (actual)\n", fk.name, fi.Name)
@@ -121,9 +146,11 @@ func idxck(ldb *db.Lowlevel) (success bool) {
 		}
 
 		if fk.device == localDeviceKey {
-			name, ok := sequences[sequenceKey{fk.folder, uint64(fi.Sequence)}]
+			sk := sequenceKey{fk.folder, uint64(fi.Sequence)}
+			name, ok := sequences[sk]
 			if !ok {
 				fmt.Printf("Sequence entry missing for FileInfo %q, folder %q, seq %d\n", fi.Name, folder, fi.Sequence)
+				missingSeq = append(missingSeq, sk)
 				success = false
 				continue
 			}
@@ -132,6 +159,58 @@ func idxck(ldb *db.Lowlevel) (success bool) {
 				success = false
 			}
 		}
+
+		if len(fi.Blocks) == 0 && len(fi.BlocksHash) != 0 {
+			key := string(fi.BlocksHash)
+			if _, ok := blocklists[key]; !ok {
+				fmt.Printf("Missing block list for file %q, block list hash %x\n", fi.Name, fi.BlocksHash)
+				success = false
+			} else {
+				usedBlocklists[key] = struct{}{}
+			}
+		}
+
+		if fi.VersionHash != nil {
+			key := string(fi.VersionHash)
+			if _, ok := versions[key]; !ok {
+				fmt.Printf("Missing version vector for file %q, version hash %x\n", fi.Name, fi.VersionHash)
+				success = false
+			} else {
+				usedVersions[key] = struct{}{}
+			}
+		}
+
+		_, ok := globals[globalKey{fk.folder, fk.name}]
+		if !ok {
+			fmt.Printf("Missing global for file %q\n", fi.Name)
+			success = false
+			continue
+		}
+	}
+
+	// Aggregate the ranges of missing sequence entries, print them
+
+	sort.Slice(missingSeq, func(a, b int) bool {
+		if missingSeq[a].folder != missingSeq[b].folder {
+			return missingSeq[a].folder < missingSeq[b].folder
+		}
+		return missingSeq[a].sequence < missingSeq[b].sequence
+	})
+
+	var folder uint32
+	var startSeq, prevSeq uint64
+	for _, sk := range missingSeq {
+		if folder != sk.folder || sk.sequence != prevSeq+1 {
+			if folder != 0 {
+				fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
+			}
+			startSeq = sk.sequence
+			folder = sk.folder
+		}
+		prevSeq = sk.sequence
+	}
+	if folder != 0 {
+		fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
 	}
 
 	for gk, vl := range globals {
@@ -140,10 +219,10 @@ func idxck(ldb *db.Lowlevel) (success bool) {
 			fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
 			success = false
 		}
-		for i, fv := range vl.Versions {
+		checkGlobal := func(i int, device []byte, version protocol.Vector, invalid, deleted bool) {
-			dev, ok := deviceToIDs[string(fv.Device)]
+			dev, ok := deviceToIDs[string(device)]
 			if !ok {
-				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, fv.Device)
+				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, device)
 				success = false
 			}
 			fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
@@ -151,14 +230,31 @@ func idxck(ldb *db.Lowlevel) (success bool) {
 				fmt.Printf("VersionList %q, folder %q, entry %d refers to unknown FileInfo\n", gk.name, folder, i)
 				success = false
 			}
-			if !fi.Version.Equal(fv.Version) {
+
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Version, fi.Version)
+			fiv := fi.Version
+			if fi.VersionHash != nil {
+				fiv = versions[string(fi.VersionHash)]
+			}
+			if !fiv.Equal(version) {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, version, fi.Version)
 				success = false
 			}
-			if fi.IsInvalid() != fv.Invalid {
+			if fi.IsInvalid() != invalid {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Invalid, fi.IsInvalid())
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, invalid, fi.IsInvalid())
 				success = false
 			}
+			if fi.IsDeleted() != deleted {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo deleted mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, deleted, fi.IsDeleted())
+				success = false
+			}
+		}
+		for i, fv := range vl.RawVersions {
+			for _, device := range fv.Devices {
+				checkGlobal(i, device, fv.Version, false, fv.Deleted)
+			}
+			for _, device := range fv.InvalidDevices {
+				checkGlobal(i, device, fv.Version, true, fv.Deleted)
+			}
 		}
 
 		// If we need this file we should have a need entry for it. False
@@ -167,7 +263,9 @@ func idxck(ldb *db.Lowlevel) (success bool) {
 		if needsLocally(vl) {
 			_, ok := needs[gk]
 			if !ok {
-				dev := deviceToIDs[string(vl.Versions[0].Device)]
+				fv, _ := vl.GetGlobal()
+				devB, _ := fv.FirstDevice()
+				dev := deviceToIDs[string(devB)]
 				fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
 				if !fi.IsDeleted() && !fi.IsIgnored() {
 					fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
@@ -224,19 +322,21 @@ func idxck(ldb *db.Lowlevel) (success bool) {
 		}
 	}
 
+	if d := len(blocklists) - len(usedBlocklists); d > 0 {
+		fmt.Printf("%d block list entries out of %d needs GC\n", d, len(blocklists))
+	}
+	if d := len(versions) - len(usedVersions); d > 0 {
+		fmt.Printf("%d version entries out of %d needs GC\n", d, len(versions))
+	}
+
 	return
 }
 
 func needsLocally(vl db.VersionList) bool {
-	var lv *protocol.Vector
+	gfv, gok := vl.GetGlobal()
-	for _, fv := range vl.Versions {
+	if !gok { // That's weird, but we hardly need something non-existant
-		if bytes.Equal(fv.Device, protocol.LocalDeviceID[:]) {
+		return false
-			lv = &fv.Version
-			break
-		}
 	}
-	if lv == nil {
+	fv, ok := vl.Get(protocol.LocalDeviceID[:])
-		return true // proviosinally, it looks like we need the file
+	return db.Need(gfv, ok, fv.Version)
-	}
-	return !lv.GreaterEqual(vl.Versions[0].Version)
 }

cmd/stindex/main.go

@@ -13,7 +13,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/db/backend"
 )
 
 func main() {
@@ -30,20 +30,23 @@ func main() {
 		path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
 	}
 
-	ldb, err := db.OpenRO(path)
+	ldb, err := backend.OpenLevelDBRO(path)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	if mode == "dump" {
+	switch mode {
+	case "dump":
 		dump(ldb)
-	} else if mode == "dumpsize" {
+	case "dumpsize":
 		dumpsize(ldb)
-	} else if mode == "idxck" {
+	case "idxck":
 		if !idxck(ldb) {
 			os.Exit(1)
 		}
-	} else {
+	case "account":
+		account(ldb)
+	default:
 		fmt.Println("Unknown mode")
 	}
 }

cmd/strelaypoolsrv/auto/noassets.go

@@ -1,10 +1,15 @@
-// Copyright (C) 2016 The Syncthing Authors.
+// Copyright (C) 2021 The Syncthing Authors.
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// The existence of this file means we get 0% test coverage rather than no
+//+build noassets
-// test coverage at all. Remove when implementing an actual test.
 
-package stats
+package auto
+
+import "github.com/syncthing/syncthing/lib/assets"
+
+func Assets() map[string]assets.Asset {
+	return nil
+}

cmd/strelaypoolsrv/gui/index.html

@@ -9,8 +9,15 @@
     <meta name="author" content=""/>
 
     <title>Relay stats</title>
-    <link href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
+    <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"/>
-    <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.0.13/css/all.css"/>
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
+    <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"/>
+    <link rel="stylesheet" href="https://unpkg.com/leaflet@1.6.0/dist/leaflet.css"
+  integrity="sha512-xwE/Az9zrjBIphAcBb3F6JVqxf46+CDLwfLMHloNu6KEQCAWi6HcDUbeOfBIptF7tcCzusKFjFw2yuvEpDL9wQ=="
+  crossorigin=""/>
+  <script src="https://unpkg.com/leaflet@1.6.0/dist/leaflet.js"
+  integrity="sha512-gZwIG9x3wUXg2hdXF6+rVkLF/0Vi9U8D2Ntg4Ga5I5BZpVkVxlJWbSQtXPSiUTtC0TjtGOmxa1AJPuV0CPthew=="
+  crossorigin=""></script>
 
     <style>
       #map {
@@ -38,13 +45,15 @@
     <div class="container">
       <h1>Relay Pool Data</h1>
       <div ng-if="relays === undefined" class="text-center">
-        <img src="//cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
+        <img src="https://cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
         <p>Please wait while we gather data</p>
       </div>
       <div>
         <div ng-show="relays !== undefined" class="ng-hide">
           <p>
-            Currently {{ relays.length }} relays online ({{ totals.goMaxProcs }} cores in total).
+            The relays listed on this page are not managed or vetted by the Syncthing project.
+            Each relay is the responsibility of the relay operator.
+            Currently {{ relays.length }} relays online.
           </p>
         </div>
         <div id="map"></div> <!-- Can't hide the map, otherwise it freaks out -->
@@ -184,18 +193,17 @@
     </div>
 
 
-    <script type="text/javascript" src="//code.jquery.com/jquery-2.1.4.min.js"></script>
+    <script type="text/javascript" src="https://code.jquery.com/jquery-2.1.4.min.js"></script>
-    <script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
+    <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
-    <script type="text/javascript" src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
+    <script type="text/javascript" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
-    <script type="text/javascript" src="//maps.googleapis.com/maps/api/js?key=AIzaSyDk5WJ8s7ueLKb99X5DbQ-vkWtPDAKqYs0"></script>
   </body>
 
   <script>
   angular.module('syncthing', [
   ])
-  .config(function($httpProvider) {
+  .config(['$httpProvider', function($httpProvider) {
     $httpProvider.defaults.timeout = 5000;
-  })
+  }])
   .filter('bytes', function() {
     return function(bytes, precision) {
       if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) return '-';
@@ -228,11 +236,12 @@
       numProxies: 0,
       uptimeSeconds: 0,
     };
-    $scope.map = new google.maps.Map(document.getElementById('map'), {
+    $scope.map = L.map('map').setView([40.90296, 1.90925], 2);
-      zoom: 1,
+    L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',
-      mapTypeId: google.maps.MapTypeId.ROADMAP
+    {
-    });
+      attribution: 'Leaflet',
-    $scope.mapBounds = new google.maps.LatLngBounds();
+      maxZoom: 17
+    }).addTo($scope.map);
     $scope.tooltipTemplate = $('#infoTemplate').html();
     $scope.usedLocations = {};
     $scope.sortType = 'stats.numActiveSessions';
@@ -279,8 +288,9 @@
         }
       });
 
-      $scope.map.fitBounds($scope.mapBounds);
       if ($scope.relays.length == 1) {
+        //Center to only relay with zoom
+        $scope.map.panTo(new L.LatLng(relays[0].location.latitude, relays[0].location.longitude));
         $scope.map.setZoom(13);
       }
     });
@@ -300,44 +310,50 @@
 
         var locParts = loc.split(',');
 
-        relay.marker = new google.maps.Marker({
+        relay.marker = new L.Marker([relay.location.latitude, relay.location.longitude],{
-          map: $scope.map,
-          position: new google.maps.LatLng(locParts[0], locParts[1]),
           title: relay.url,
         });
 
         var scope = $rootScope.$new(true);
         scope.relay = relay;
 
-        relay.marker.info = new google.maps.InfoWindow({
+        var icon = new L.Icon({
-          content: $compile($scope.tooltipTemplate)(scope)[0],
+          iconSize:     [18, 28], // size of the icon
+          iconAnchor:   [9, 28], // point of the icon which will correspond to marker's location
+          shadowAnchor: [0, 0],  // the same for the shadow
+          popupAnchor:  [0, -27], // popup anchor
+          shadowSize:   [0,0],
+          iconUrl: 'https://cdn.rawgit.com/pointhi/leaflet-color-markers/master/img/marker-icon-red.png',
+          shadowUrl: 'https://cdnjs.cloudflare.com/ajax/libs/leaflet/0.7.7/images/marker-shadow.png',
         });
 
+        relay.marker = new L.marker(new L.latLng(locParts[0], locParts[1]),{icon})
+                            .bindPopup($compile($scope.tooltipTemplate)(scope)[0],{})
+                            .on('mouseover', function (e) {
+                                this.openPopup();
+                            }).on('mouseout', function (e) {
+                                this.closePopup();
+                            }).addTo($scope.map);
+
         relay.showMarker = function() {
-          relay.marker.info.open($scope.map, relay.marker);
+          relay.marker.openPopup();
         }
-
+        
         relay.hideMarker = function() {
-          relay.marker.info.close();
+          relay.marker.closePopup();
         }
+      }
 
-        relay.marker.addListener('mouseover', relay.showMarker);
-        relay.marker.addListener('mouseout', relay.hideMarker);
-
-        $scope.mapBounds.extend(relay.marker.position);
-    }
 
     function addCircleToMap(relay) {
-      relay.marker.circle = new google.maps.Circle({
+      console.log(relay.location.latitude)
-        strokeColor: '#FF0000',
+      L.circle([relay.location.latitude, relay.location.longitude], 
-        strokeOpacity: 0.8,
+        {
-        strokeWeight: 2,
+          radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000,
-        fillColor: '#FF0000',
+          color: "FF0000",
-        fillOpacity: 0.35,
+          fillColor: "#FF0000",
-        map: $scope.map,
+          fillOpacity: 0.35,
-        center: relay.marker.position,
+        }).addTo($scope.map);
-        radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000
-      });
     }
 
     function constructURI(url) {

cmd/strelaypoolsrv/main.go

@@ -1,20 +1,18 @@
 // Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
 
-//go:generate go run ../../script/genassets.go gui >auto/gui.go
-
 package main
 
 import (
-	"bytes"
 	"compress/gzip"
+	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
-	"math/rand"
-	"mime"
 	"net"
 	"net/http"
 	"net/url"
@@ -24,11 +22,15 @@ import (
 	"strings"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/protocol"
+
 	"github.com/golang/groupcache/lru"
 	"github.com/oschwald/geoip2-golang"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
+	"github.com/syncthing/syncthing/lib/assets"
+	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/relay/client"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -92,33 +94,35 @@ type result struct {
 }
 
 var (
-	testCert        tls.Certificate
+	testCert          tls.Certificate
-	knownRelaysFile = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
+	knownRelaysFile   = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
-	listen          = ":80"
+	listen            = ":80"
-	dir             string
+	dir               string
-	evictionTime    = time.Hour
+	evictionTime      = time.Hour
-	debug           bool
+	debug             bool
-	getLRUSize      = 10 << 10
+	getLRUSize        = 10 << 10
-	getLimitBurst   = 10
+	getLimitBurst     = 10
-	getLimitAvg     = 2
+	getLimitAvg       = 2
-	postLRUSize     = 1 << 10
+	postLRUSize       = 1 << 10
-	postLimitBurst  = 2
+	postLimitBurst    = 2
-	postLimitAvg    = 2
+	postLimitAvg      = 2
-	getLimit        time.Duration
+	getLimit          time.Duration
-	postLimit       time.Duration
+	postLimit         time.Duration
-	permRelaysFile  string
+	permRelaysFile    string
-	ipHeader        string
+	ipHeader          string
-	geoipPath       string
+	geoipPath         string
-	proto           string
+	proto             string
-	statsRefresh    = time.Minute / 2
+	statsRefresh      = time.Minute / 2
+	requestQueueLen   = 10
+	requestProcessors = 1
 
-	getMut      = sync.NewRWMutex()
+	getMut      = sync.NewMutex()
 	getLRUCache *lru.Cache
 
-	postMut      = sync.NewRWMutex()
+	postMut      = sync.NewMutex()
 	postLRUCache *lru.Cache
 
-	requests = make(chan request, 10)
+	requests chan request
 
 	mut             = sync.NewRWMutex()
 	knownRelays     = make([]*relay, 0)
@@ -131,6 +135,9 @@ const (
 )
 
 func main() {
+	log.SetOutput(os.Stdout)
+	log.SetFlags(log.Lshortfile)
+
 	flag.StringVar(&listen, "listen", listen, "Listen address")
 	flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
 	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
@@ -146,9 +153,13 @@ func main() {
 	flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
 	flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
 	flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
+	flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
+	flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
 
 	flag.Parse()
 
+	requests = make(chan request, requestQueueLen)
+
 	getLimit = 10 * time.Second / time.Duration(getLimitAvg)
 	postLimit = time.Minute / time.Duration(postLimitAvg)
 
@@ -164,7 +175,9 @@ func main() {
 
 	testCert = createTestCertificate()
 
-	go requestProcessor()
+	for i := 0; i < requestProcessors; i++ {
+		go requestProcessor()
+	}
 
 	// Load relays from cache in the background.
 	// Load them in a serial fashion to make sure any genuine requests
@@ -198,6 +211,7 @@ func main() {
 		tlsCfg := &tls.Config{
 			Certificates: []tls.Certificate{cert},
 			MinVersion:   tls.VersionTLS10, // No SSLv3
+			ClientAuth:   tls.RequestClientCert,
 			CipherSuites: []uint16{
 				// No RC4
 				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
@@ -253,88 +267,27 @@ func handleMetrics(w http.ResponseWriter, r *http.Request) {
 func handleAssets(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Cache-Control", "no-cache, must-revalidate")
 
-	assets := auto.Assets()
 	path := r.URL.Path[1:]
 	if path == "" {
 		path = "index.html"
 	}
 
-	bs, ok := assets[path]
+	as, ok := auto.Assets()[path]
 	if !ok {
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
 
-	etag := fmt.Sprintf("%d", auto.Generated)
+	assets.Serve(w, r, as)
-	modified := time.Unix(auto.Generated, 0).UTC()
-
-	w.Header().Set("Last-Modified", modified.Format(http.TimeFormat))
-	w.Header().Set("Etag", etag)
-
-	mtype := mimeTypeForFile(path)
-	if len(mtype) != 0 {
-		w.Header().Set("Content-Type", mtype)
-	}
-
-	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modified.Add(time.Second).After(t) {
-		w.WriteHeader(http.StatusNotModified)
-		return
-	}
-
-	if match := r.Header.Get("If-None-Match"); match != "" {
-		if strings.Contains(match, etag) {
-			w.WriteHeader(http.StatusNotModified)
-			return
-		}
-	}
-	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
-		w.Header().Set("Content-Encoding", "gzip")
-	} else {
-		// ungzip if browser not send gzip accepted header
-		var gr *gzip.Reader
-		gr, _ = gzip.NewReader(bytes.NewReader(bs))
-		bs, _ = ioutil.ReadAll(gr)
-		gr.Close()
-	}
-	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs)))
-
-	w.Write(bs)
-}
-
-func mimeTypeForFile(file string) string {
-	// We use a built in table of the common types since the system
-	// TypeByExtension might be unreliable. But if we don't know, we delegate
-	// to the system.
-	ext := filepath.Ext(file)
-	switch ext {
-	case ".htm", ".html":
-		return "text/html"
-	case ".css":
-		return "text/css"
-	case ".js":
-		return "application/javascript"
-	case ".json":
-		return "application/json"
-	case ".png":
-		return "image/png"
-	case ".ttf":
-		return "application/x-font-ttf"
-	case ".woff":
-		return "application/x-font-woff"
-	case ".svg":
-		return "image/svg+xml"
-	default:
-		return mime.TypeByExtension(ext)
-	}
 }
 
 func handleRequest(w http.ResponseWriter, r *http.Request) {
 	timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
 
-	lw := NewLoggingResponseWriter(w)
+	w = NewLoggingResponseWriter(w)
-
 	defer func() {
 		timer.ObserveDuration()
+		lw := w.(*loggingResponseWriter)
 		apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
 	}()
 
@@ -363,24 +316,38 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func handleGetRequest(w http.ResponseWriter, r *http.Request) {
+func handleGetRequest(rw http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
+
 	mut.RLock()
-	relays := append(permanentRelays, knownRelays...)
+	relays := make([]*relay, len(permanentRelays)+len(knownRelays))
+	n := copy(relays, permanentRelays)
+	copy(relays[n:], knownRelays)
 	mut.RUnlock()
 
 	// Shuffle
-	for i := range relays {
+	rand.Shuffle(relays)
-		j := rand.Intn(i + 1)
+
-		relays[i], relays[j] = relays[j], relays[i]
+	w := io.Writer(rw)
+	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+		rw.Header().Set("Content-Encoding", "gzip")
+		gw := gzip.NewWriter(rw)
+		defer gw.Close()
+		w = gw
 	}
 
-	json.NewEncoder(w).Encode(map[string][]*relay{
+	_ = json.NewEncoder(w).Encode(map[string][]*relay{
 		"relays": relays,
 	})
 }
 
 func handlePostRequest(w http.ResponseWriter, r *http.Request) {
+	var relayCert *x509.Certificate
+	if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
+		relayCert = r.TLS.PeerCertificates[0]
+		log.Printf("Got TLS cert from relay server")
+	}
+
 	var newRelay relay
 	err := json.NewDecoder(r.Body).Decode(&newRelay)
 	r.Body.Close()
@@ -389,7 +356,7 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		if debug {
 			log.Println("Failed to parse payload")
 		}
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
@@ -398,16 +365,26 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		if debug {
 			log.Println("Failed to parse URI", newRelay.URL)
 		}
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
+	if relayCert != nil {
+		advertisedId := uri.Query().Get("id")
+		idFromCert := protocol.NewDeviceID(relayCert.Raw).String()
+		if advertisedId != idFromCert {
+			log.Println("Warning: Relay server requested to join with an ID different from the join request, rejecting")
+			http.Error(w, "mismatched advertised id and join request cert", http.StatusBadRequest)
+			return
+		}
+	}
+
 	host, port, err := net.SplitHostPort(uri.Host)
 	if err != nil {
 		if debug {
 			log.Println("Failed to split URI", newRelay.URL)
 		}
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
@@ -422,7 +399,7 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 	if ip == nil || ip.IsUnspecified() {
 		uri.Host = net.JoinHostPort(rhost, port)
 		newRelay.URL = uri.String()
-	} else if host != rhost {
+	} else if host != rhost && relayCert == nil {
 		if debug {
 			log.Println("IP address advertised does not match client IP address", r.RemoteAddr, uri)
 		}
@@ -483,11 +460,11 @@ func handleRelayTest(request request) {
 	if debug {
 		log.Println("Request for", request.relay)
 	}
-	if !client.TestRelay(request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
+	if err := client.TestRelay(context.TODO(), request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3); err != nil {
 		if debug {
-			log.Println("Test for relay", request.relay, "failed")
+			log.Println("Test for relay", request.relay, "failed:", err)
 		}
-		request.result <- result{fmt.Errorf("connection test failed"), 0}
+		request.result <- result{err, 0}
 		return
 	}
 
@@ -499,7 +476,7 @@ func handleRelayTest(request request) {
 		updateMetrics(request.relay.uri.Host, *stats, location)
 	}
 	request.relay.Stats = stats
-	request.relay.StatsRetrieved = time.Now()
+	request.relay.StatsRetrieved = time.Now().Truncate(time.Second)
 	request.relay.Location = location
 
 	timer, ok := evictionTimers[request.relay.uri.Host]
@@ -565,26 +542,21 @@ func evict(relay *relay) func() {
 	}
 }
 
-func limit(addr string, cache *lru.Cache, lock sync.RWMutex, intv time.Duration, burst int) bool {
+func limit(addr string, cache *lru.Cache, lock sync.Mutex, intv time.Duration, burst int) bool {
 	if host, _, err := net.SplitHostPort(addr); err == nil {
 		addr = host
 	}
 
-	lock.RLock()
+	lock.Lock()
-	bkt, ok := cache.Get(addr)
+	v, _ := cache.Get(addr)
-	lock.RUnlock()
+	bkt, ok := v.(*rate.Limiter)
-	if ok {
+	if !ok {
-		bkt := bkt.(*rate.Limiter)
+		bkt = rate.NewLimiter(rate.Every(intv), burst)
-		if !bkt.Allow() {
+		cache.Add(addr, bkt)
-			// Rate limit
-			return true
-		}
-	} else {
-		lock.Lock()
-		cache.Add(addr, rate.NewLimiter(rate.Every(intv), burst))
-		lock.Unlock()
 	}
-	return false
+	lock.Unlock()
+
+	return !bkt.Allow()
 }
 
 func loadRelays(file string) []*relay {
@@ -636,7 +608,7 @@ func createTestCertificate() tls.Certificate {
 	}
 
 	certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
-	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv")
+	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 20*365)
 	if err != nil {
 		log.Fatalln("Failed to create test X509 key pair:", err)
 	}

cmd/strelaypoolsrv/main_test.go

@@ -0,0 +1,67 @@
+// Copyright © 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+)
+
+func init() {
+	for i := 0; i < 10; i++ {
+		u := fmt.Sprintf("permanent%d", i)
+		permanentRelays = append(permanentRelays, &relay{URL: u})
+	}
+
+	knownRelays = []*relay{
+		{URL: "known1"},
+		{URL: "known2"},
+		{URL: "known3"},
+	}
+
+	mut = new(sync.RWMutex)
+}
+
+// Regression test: handleGetRequest should not modify permanentRelays.
+func TestHandleGetRequest(t *testing.T) {
+	needcap := len(permanentRelays) + len(knownRelays)
+	if needcap > cap(permanentRelays) {
+		t.Fatalf("test setup failed: need cap(permanentRelays) >= %d, have %d",
+			needcap, cap(permanentRelays))
+	}
+
+	w := httptest.NewRecorder()
+	w.Body = new(bytes.Buffer)
+	handleGetRequest(w, httptest.NewRequest("GET", "/", nil))
+
+	result := make(map[string][]*relay)
+	err := json.NewDecoder(w.Body).Decode(&result)
+	if err != nil {
+		t.Fatalf("invalid JSON: %v", err)
+	}
+
+	relays := result["relays"]
+	expect, actual := len(knownRelays)+len(permanentRelays), len(relays)
+	if actual != expect {
+		t.Errorf("expected %d relays, got %d", expect, actual)
+	}
+
+	// Check for changes in permanentRelays.
+	for i, r := range permanentRelays {
+		switch {
+		case !strings.HasPrefix(r.URL, "permanent"):
+			t.Errorf("relay %q among permanent relays", r.URL)
+		case r.URL != fmt.Sprintf("permanent%d", i):
+			t.Error("order of permanent relays changed")
+		}
+	}
+}

cmd/strelaysrv/README.md

@@ -121,11 +121,11 @@ Relay related libraries used by this repo
 ----
 ##### Relay protocol definition.
 
-[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/protocol)
+[Available here](https://github.com/syncthing/syncthing/tree/main/lib/relay/protocol)
 
 
 ##### Relay client
 
 Only used by the testutil.
 
-[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/client)
+[Available here](https://github.com/syncthing/syncthing/tree/main/lib/relay/client)

cmd/strelaysrv/etc/firewall-ufw/strelaysrv

@@ -0,0 +1,9 @@
+[strelaysrv]
+title=Syncthing relay server
+description=Proxies traffic of syncthing client behind firewalls
+ports=22067/tcp
+
+[strelaysrv-metrics]
+title=Syncthing relay metrics
+description=Provides metrics about the syncthing relay server
+ports=22070/tcp

cmd/strelaysrv/etc/linux-systemd/default

@@ -0,0 +1,5 @@
+# Default settings for syncthing-relaysrv (strelaysrv).
+NAT=true
+
+## Add Options here:
+RELAYSRV_OPTS=

cmd/strelaysrv/etc/linux-systemd/strelaysrv.service

@@ -1,17 +1,25 @@
 [Unit]
-Description=Syncthing relay server
+Description=Syncthing Relay Server
 After=network.target
+Documentation=man:strelaysrv(1)
 
 [Service]
-User=strelaysrv
+WorkingDirectory=/var/lib/syncthing-relaysrv
-Group=strelaysrv
+EnvironmentFile=/etc/default/syncthing-relaysrv
-ExecStart=/usr/bin/strelaysrv
+ExecStart=/usr/bin/strelaysrv -nat=${NAT} $RELAYSRV_OPTS
-WorkingDirectory=/var/lib/strelaysrv
 
-PrivateTmp=true
+# Hardening
-ProtectSystem=full
+User=syncthing-relaysrv
-ProtectHome=true
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-relaysrv
 NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
 
 [Install]
 WantedBy=multi-user.target
+Alias=syncthing-relaysrv.service

cmd/strelaysrv/listener.go

@@ -149,7 +149,15 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
 				protocol.WriteMessage(conn, protocol.ResponseSuccess)
 
 			case protocol.ConnectRequest:
-				requestedPeer := syncthingprotocol.DeviceIDFromBytes(msg.ID)
+				requestedPeer, err := syncthingprotocol.DeviceIDFromBytes(msg.ID)
+				if err != nil {
+					if debug {
+						log.Println(id, "is looking for an invalid peer ID")
+					}
+					protocol.WriteMessage(conn, protocol.ResponseNotFound)
+					conn.Close()
+					continue
+				}
 				outboxesMut.RLock()
 				peerOutbox, ok := outboxes[requestedPeer]
 				outboxesMut.RUnlock()

cmd/strelaysrv/main.go

@@ -3,6 +3,7 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
 	"fmt"
@@ -14,12 +15,13 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
 	"sync/atomic"
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/relay/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -33,24 +35,6 @@ import (
 	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
 )
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`strelaysrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
-}
-
 var (
 	listen string
 	debug  bool
@@ -115,9 +99,16 @@ func main() {
 	flag.IntVar(&natRenewal, "nat-renewal", 30, "NAT renewal frequency in minutes")
 	flag.IntVar(&natTimeout, "nat-timeout", 10, "NAT discovery timeout in seconds")
 	flag.BoolVar(&pprofEnabled, "pprof", false, "Enable the built in profiling on the status server")
-	flag.IntVar(&networkBufferSize, "network-buffer", 2048, "Network buffer size (two of these per proxied connection)")
+	flag.IntVar(&networkBufferSize, "network-buffer", 65536, "Network buffer size (two of these per proxied connection)")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
+	longVer := build.LongVersionFor("strelaysrv")
+	if *showVersion {
+		fmt.Println(longVer)
+		return
+	}
+
 	if extAddress == "" {
 		extAddress = listen
 	}
@@ -146,7 +137,7 @@ func main() {
 		}
 	}
 
-	log.Println(LongVersion)
+	log.Println(longVer)
 
 	maxDescriptors, err := osutil.MaximizeOpenFileLimit()
 	if maxDescriptors > 0 {
@@ -166,7 +157,7 @@ func main() {
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
@@ -194,17 +185,20 @@ func main() {
 		log.Println("ID:", id)
 	}
 
-	wrapper := config.Wrap("config", config.New(id))
+	wrapper := config.Wrap("config", config.New(id), id, events.NoopLogger)
-	wrapper.SetOptions(config.OptionsConfiguration{
+	go wrapper.Serve(context.TODO())
-		NATLeaseM:   natLease,
+	wrapper.Modify(func(cfg *config.Configuration) {
-		NATRenewalM: natRenewal,
+		cfg.Options.NATLeaseM = natLease
-		NATTimeoutS: natTimeout,
+		cfg.Options.NATRenewalM = natRenewal
+		cfg.Options.NATTimeoutS = natTimeout
 	})
 	natSvc := nat.NewService(id, wrapper)
 	mapping := mapping{natSvc.NewMapping(nat.TCP, addr.IP, addr.Port)}
 
 	if natEnabled {
-		go natSvc.Serve()
+		ctx, cancel := context.WithCancel(context.Background())
+		go natSvc.Serve(ctx)
+		defer cancel()
 		found := make(chan struct{})
 		mapping.OnChanged(func(_ *nat.Mapping, _, _ []nat.Address) {
 			select {
@@ -239,6 +233,7 @@ func main() {
 	uri, err := url.Parse(fmt.Sprintf("relay://%s/?id=%s&pingInterval=%s&networkTimeout=%s&sessionLimitBps=%d&globalLimitBps=%d&statusAddr=%s&providedBy=%s", mapping.Address(), id, pingInterval, networkTimeout, sessionLimitBps, globalLimitBps, statusAddr, providedBy))
 	if err != nil {
 		log.Fatalln("Failed to construct URI", err)
+		return
 	}
 
 	log.Println("URI:", uri.String())
@@ -254,7 +249,7 @@ func main() {
 	for _, pool := range pools {
 		pool = strings.TrimSpace(pool)
 		if len(pool) > 0 {
-			go poolHandler(pool, uri, mapping)
+			go poolHandler(pool, uri, mapping, cert)
 		}
 	}
 

cmd/strelaysrv/pool.go

@@ -4,14 +4,20 @@ package main
 
 import (
 	"bytes"
+	"crypto/tls"
 	"encoding/json"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"net/url"
 	"time"
 )
 
-func poolHandler(pool string, uri *url.URL, mapping mapping) {
+const (
+	httpStatusEnhanceYourCalm = 429
+)
+
+func poolHandler(pool string, uri *url.URL, mapping mapping, ownCert tls.Certificate) {
 	if debug {
 		log.Println("Joining", pool)
 	}
@@ -26,40 +32,81 @@ func poolHandler(pool string, uri *url.URL, mapping mapping) {
 			uriCopy.String(),
 		})
 
-		resp, err := httpClient.Post(pool, "application/json", &b)
+		poolUrl, err := url.Parse(pool)
 		if err != nil {
-			log.Println("Error joining pool", pool, err)
+			log.Printf("Could not parse pool url '%s': %v", pool, err)
-		} else if resp.StatusCode == 500 {
+		}
-			bs, err := ioutil.ReadAll(resp.Body)
+
-			if err != nil {
+		client := http.DefaultClient
-				log.Println("Failed to join", pool, "due to an internal server error. Could not read response:", err)
+		if poolUrl.Scheme == "https" {
-			} else {
+			// Sent our certificate in join request
-				log.Println("Failed to join", pool, "due to an internal server error:", string(bs))
+			client = &http.Client{
+				Transport: &http.Transport{
+					TLSClientConfig: &tls.Config{
+						Certificates: []tls.Certificate{ownCert},
+					},
+				},
 			}
-			resp.Body.Close()
+		}
-		} else if resp.StatusCode == 429 {
+
-			log.Println(pool, "under load, will retry in a minute")
+		resp, err := client.Post(pool, "application/json", &b)
+		if err != nil {
+			log.Printf("Error joining pool %v: HTTP request: %v", pool, err)
 			time.Sleep(time.Minute)
 			continue
-		} else if resp.StatusCode == 401 {
+		}
-			log.Println(pool, "failed to join due to IP address not matching external address. Aborting")
+
-			return
+		bs, err := ioutil.ReadAll(resp.Body)
-		} else if resp.StatusCode == 200 {
+		resp.Body.Close()
+		if err != nil {
+			log.Printf("Error joining pool %v: reading response: %v", pool, err)
+			time.Sleep(time.Minute)
+			continue
+		}
+
+		switch resp.StatusCode {
+		case http.StatusOK:
 			var x struct {
 				EvictionIn time.Duration `json:"evictionIn"`
 			}
-			err := json.NewDecoder(resp.Body).Decode(&x)
+			if err := json.Unmarshal(bs, &x); err == nil {
-			if err == nil {
 				rejoin := x.EvictionIn - (x.EvictionIn / 5)
-				log.Println("Joined", pool, "rejoining in", rejoin)
+				log.Printf("Joined pool %s, rejoining in %v", pool, rejoin)
 				time.Sleep(rejoin)
 				continue
 			} else {
-				log.Println("Failed to deserialize response", err)
+				log.Printf("Joined pool %s, failed to deserialize response: %v", pool, err)
 			}
-		} else {
+
-			log.Println(pool, "unknown response type from server", resp.StatusCode)
+		case http.StatusInternalServerError:
+			log.Printf("Failed to join %v: server error", pool)
+			log.Printf("Response data: %s", bs)
+			time.Sleep(time.Minute)
+			continue
+
+		case http.StatusBadRequest:
+			log.Printf("Failed to join %v: request or check error", pool)
+			log.Printf("Response data: %s", bs)
+			time.Sleep(time.Minute)
+			continue
+
+		case httpStatusEnhanceYourCalm:
+			log.Printf("Failed to join %v: under load (rate limiting)", pool)
+			time.Sleep(time.Minute)
+			continue
+
+		case http.StatusUnauthorized:
+			log.Printf("Failed to join %v: IP address not matching external address", pool)
+			log.Println("Aborting")
+			return
+
+		default:
+			log.Printf("Failed to join %v: unexpected status code from server: %d", pool, resp.StatusCode)
+			log.Printf("Response data: %s", bs)
+			time.Sleep(time.Minute)
+			continue
 		}
+
 		time.Sleep(time.Hour)
 	}
 }

cmd/strelaysrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-relaysrv --ingroup syncthing syncthing-relaysrv

cmd/strelaysrv/status.go

@@ -10,6 +10,8 @@ import (
 	"runtime"
 	"sync/atomic"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/build"
 )
 
 var rc *rateCalculator
@@ -40,10 +42,10 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 
 	sessionMut.Lock()
 	// This can potentially be double the number of pending sessions, as each session has two keys, one for each side.
-	status["version"] = Version
+	status["version"] = build.Version
-	status["buildHost"] = BuildHost
+	status["buildHost"] = build.Host
-	status["buildUser"] = BuildUser
+	status["buildUser"] = build.User
-	status["buildDate"] = BuildDate
+	status["buildDate"] = build.Date
 	status["startTime"] = rc.startTime
 	status["uptimeSeconds"] = time.Since(rc.startTime) / time.Second
 	status["numPendingSessionKeys"] = len(pendingSessions)
@@ -86,9 +88,9 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 }
 
 type rateCalculator struct {
+	counter   *int64 // atomic, must remain 64-bit aligned
 	rates     []int64
 	prev      int64
-	counter   *int64
 	startTime time.Time
 }
 

cmd/strelaysrv/testutil/main.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"crypto/tls"
 	"flag"
 	"log"
@@ -19,6 +20,9 @@ import (
 )
 
 func main() {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	log.SetOutput(os.Stdout)
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
 
@@ -59,7 +63,7 @@ func main() {
 		}
 		log.Println("Created client")
 
-		go relay.Serve()
+		go relay.Serve(ctx)
 
 		recv := make(chan protocol.SessionInvitation)
 
@@ -76,7 +80,7 @@ func main() {
 		}()
 
 		for {
-			conn, err := client.JoinSession(<-recv)
+			conn, err := client.JoinSession(ctx, <-recv)
 			if err != nil {
 				log.Fatalln("Failed to join", err)
 			}
@@ -90,13 +94,13 @@ func main() {
 			log.Fatal(err)
 		}
 
-		invite, err := client.GetInvitationFromRelay(uri, id, []tls.Certificate{cert}, 10*time.Second)
+		invite, err := client.GetInvitationFromRelay(ctx, uri, id, []tls.Certificate{cert}, 10*time.Second)
 		if err != nil {
 			log.Fatal(err)
 		}
 
 		log.Println("Received invitation", invite)
-		conn, err := client.JoinSession(invite)
+		conn, err := client.JoinSession(ctx, invite)
 		if err != nil {
 			log.Fatalln("Failed to join", err)
 		}
@@ -104,10 +108,10 @@ func main() {
 		connectToStdio(stdin, conn)
 		log.Println("Finished", conn.RemoteAddr(), conn.LocalAddr())
 	} else if test {
-		if client.TestRelay(uri, []tls.Certificate{cert}, time.Second, 2*time.Second, 4) {
+		if err := client.TestRelay(ctx, uri, []tls.Certificate{cert}, time.Second, 2*time.Second, 4); err == nil {
 			log.Println("OK")
 		} else {
-			log.Println("FAIL")
+			log.Println("FAIL:", err)
 		}
 	} else {
 		log.Fatal("Requires either join or connect")

cmd/stupgrades/main.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"os"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/upgrade"
+)
+
+const defaultURL = "https://api.github.com/repos/syncthing/syncthing/releases?per_page=25"
+
+func main() {
+	url := flag.String("u", defaultURL, "GitHub releases url")
+	flag.Parse()
+
+	rels := upgrade.FetchLatestReleases(*url, "")
+	if rels == nil {
+		// An error was already logged
+		os.Exit(1)
+	}
+
+	sort.Sort(upgrade.SortByRelease(rels))
+	rels = filterForLatest(rels)
+
+	if err := json.NewEncoder(os.Stdout).Encode(rels); err != nil {
+		os.Exit(1)
+	}
+}
+
+// filterForLatest returns the latest stable and prerelease only. If the
+// stable version is newer (comes first in the list) there is no need to go
+// looking for a prerelease at all.
+func filterForLatest(rels []upgrade.Release) []upgrade.Release {
+	var filtered []upgrade.Release
+	var havePre bool
+	for _, rel := range rels {
+		if !rel.Prerelease {
+			// We found a stable version, we're good now.
+			filtered = append(filtered, rel)
+			break
+		}
+		if rel.Prerelease && !havePre {
+			// We remember the first prerelease we find.
+			filtered = append(filtered, rel)
+			havePre = true
+		}
+	}
+	return filtered
+}

cmd/stvanity/main.go

@@ -14,6 +14,7 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
+	"errors"
 	"flag"
 	"fmt"
 	"math/big"
@@ -36,7 +37,7 @@ type result struct {
 
 func main() {
 	flag.Parse()
-	prefix := strings.ToUpper(strings.Replace(flag.Arg(0), "-", "", -1))
+	prefix := strings.ToUpper(strings.ReplaceAll(flag.Arg(0), "-", ""))
 	if len(prefix) > 7 {
 		prefix = prefix[:7] + "-" + prefix[7:]
 	}
@@ -191,7 +192,7 @@ func pemBlockForKey(priv interface{}) (*pem.Block, error) {
 		}
 		return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}, nil
 	default:
-		return nil, fmt.Errorf("unknown key type")
+		return nil, errors.New("unknown key type")
 	}
 }
 

cmd/stwatchfile/main.go

@@ -7,31 +7,15 @@
 package main
 
 import (
-	"bytes"
-	"crypto/md5"
 	"flag"
 	"fmt"
 	"io"
 	"os"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
-func getmd5(filePath string) ([]byte, error) {
-	var result []byte
-	file, err := os.Open(filePath)
-	if err != nil {
-		return result, err
-	}
-	defer file.Close()
-
-	hash := md5.New()
-	if _, err := io.Copy(hash, file); err != nil {
-		return result, err
-	}
-
-	return hash.Sum(result), nil
-}
-
 func main() {
 	period := flag.Duration("period", 200*time.Millisecond, "Sleep period between checks")
 	flag.Parse()
@@ -46,7 +30,7 @@ func main() {
 	exists := true
 	size := int64(0)
 	mtime := time.Time{}
-	hash := []byte{}
+	var hash [sha256.Size]byte
 
 	for {
 		time.Sleep(*period)
@@ -72,7 +56,7 @@ func main() {
 		if !exists {
 			size = 0
 			mtime = time.Time{}
-			hash = []byte{}
+			hash = [sha256.Size]byte{}
 			continue
 		}
 
@@ -83,12 +67,12 @@ func main() {
 		newSize := fi.Size()
 		newMtime := fi.ModTime()
 
-		newHash, err := getmd5(file)
+		newHash, err := sha256file(file)
 		if err != nil {
-			fmt.Println("getmd5:", err)
+			fmt.Println("sha256file:", err)
 		}
 
-		if newSize != size || newMtime != mtime || !bytes.Equal(newHash, hash) {
+		if newSize != size || newMtime != mtime || newHash != hash {
 			fmt.Println(file, "Size:", newSize, "Mtime:", newMtime, "Hash:", fmt.Sprintf("%x", newHash))
 			hash = newHash
 			size = newSize
@@ -96,3 +80,18 @@ func main() {
 		}
 	}
 }
+
+func sha256file(fname string) (hash [sha256.Size]byte, err error) {
+	f, err := os.Open(fname)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	io.Copy(h, f)
+	hb := h.Sum(nil)
+	copy(hash[:], hb)
+
+	return
+}

cmd/syncthing/auditservice.go

@@ -1,69 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"encoding/json"
-	"io"
-
-	"github.com/syncthing/syncthing/lib/events"
-)
-
-// The auditService subscribes to events and writes these in JSON format, one
-// event per line, to the specified writer.
-type auditService struct {
-	w       io.Writer     // audit destination
-	stop    chan struct{} // signals time to stop
-	started chan struct{} // signals startup complete
-	stopped chan struct{} // signals stop complete
-}
-
-func newAuditService(w io.Writer) *auditService {
-	return &auditService{
-		w:       w,
-		stop:    make(chan struct{}),
-		started: make(chan struct{}),
-		stopped: make(chan struct{}),
-	}
-}
-
-// Serve runs the audit service.
-func (s *auditService) Serve() {
-	defer close(s.stopped)
-	sub := events.Default.Subscribe(events.AllEvents)
-	defer events.Default.Unsubscribe(sub)
-	enc := json.NewEncoder(s.w)
-
-	// We're ready to start processing events.
-	close(s.started)
-
-	for {
-		select {
-		case ev := <-sub.C():
-			enc.Encode(ev)
-		case <-s.stop:
-			return
-		}
-	}
-}
-
-// Stop stops the audit service.
-func (s *auditService) Stop() {
-	close(s.stop)
-}
-
-// WaitForStart returns once the audit service is ready to receive events, or
-// immediately if it's already running.
-func (s *auditService) WaitForStart() {
-	<-s.started
-}
-
-// WaitForStop returns once the audit service has stopped.
-// (Needed by the tests.)
-func (s *auditService) WaitForStop() {
-	<-s.stopped
-}

cmd/syncthing/blockprof.go

@@ -15,18 +15,20 @@ import (
 	"time"
 )
 
-func init() {
+func startBlockProfiler() {
-	if innerProcess && os.Getenv("STBLOCKPROFILE") != "" {
+	profiler := pprof.Lookup("block")
-		profiler := pprof.Lookup("block")
+	if profiler == nil {
-		if profiler == nil {
+		panic("Couldn't find block profiler")
-			panic("Couldn't find block profiler")
-		}
-		l.Debugln("Starting block profiling")
-		go saveBlockingProfiles(profiler)
 	}
+	l.Debugln("Starting block profiling")
+	go func() {
+		err := saveBlockingProfiles(profiler) // Only returns on error
+		l.Warnln("Block profiler failed:", err)
+		panic("Block profiler failed")
+	}()
 }
 
-func saveBlockingProfiles(profiler *pprof.Profile) {
+func saveBlockingProfiles(profiler *pprof.Profile) error {
 	runtime.SetBlockProfileRate(1)
 
 	t0 := time.Now()
@@ -35,16 +37,16 @@ func saveBlockingProfiles(profiler *pprof.Profile) {
 
 		fd, err := os.Create(fmt.Sprintf("block-%05d-%07d.pprof", syscall.Getpid(), startms))
 		if err != nil {
-			panic(err)
+			return err
 		}
 		err = profiler.WriteTo(fd, 0)
 		if err != nil {
-			panic(err)
+			return err
 		}
 		err = fd.Close()
 		if err != nil {
-			panic(err)
+			return err
 		}
-
 	}
+	return nil
 }

cmd/syncthing/cli/client.go

@@ -4,12 +4,13 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
 	"bytes"
 	"context"
 	"crypto/tls"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -80,16 +81,16 @@ func (c *APIClient) Post(url, body string) (*http.Response, error) {
 
 func checkResponse(response *http.Response) error {
 	if response.StatusCode == 404 {
-		return fmt.Errorf("Invalid endpoint or API call")
+		return errors.New("invalid endpoint or API call")
 	} else if response.StatusCode == 403 {
-		return fmt.Errorf("Invalid API key")
+		return errors.New("invalid API key")
 	} else if response.StatusCode != 200 {
 		data, err := responseToBArray(response)
 		if err != nil {
 			return err
 		}
 		body := strings.TrimSpace(string(data))
-		return fmt.Errorf("Unexpected HTTP status returned: %s\n%s", response.Status, body)
+		return fmt.Errorf("unexpected HTTP status returned: %s\n%s", response.Status, body)
 	}
 	return nil
 }

cmd/syncthing/cli/errors.go

@@ -4,9 +4,10 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
+	"errors"
 	"fmt"
 	"strings"
 
@@ -54,7 +55,7 @@ func errorsPush(c *cli.Context) error {
 		if body != "" {
 			errStr += "\nBody: " + body
 		}
-		return fmt.Errorf(errStr)
+		return errors.New(errStr)
 	}
 	return nil
 }

cmd/syncthing/cli/main.go

@@ -0,0 +1,236 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"bufio"
+	"crypto/tls"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"reflect"
+	"strings"
+
+	"github.com/AudriusButkevicius/recli"
+	"github.com/alecthomas/kong"
+	"github.com/flynn-archive/go-shlex"
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/locations"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/urfave/cli"
+)
+
+type preCli struct {
+	GUIAddress string `name:"gui-address"`
+	GUIAPIKey  string `name:"gui-apikey"`
+	HomeDir    string `name:"home"`
+	ConfDir    string `name:"conf"`
+}
+
+func Run() error {
+	// This is somewhat a hack around a chicken and egg problem. We need to set
+	// the home directory and potentially other flags to know where the
+	// syncthing instance is running in order to get it's config ... which we
+	// then use to construct the actual CLI ... at which point it's too late to
+	// add flags there...
+	c := preCli{}
+	parseFlags(&c)
+
+	// Not set as default above because the strings can be really long.
+	var err error
+	homeSet := c.HomeDir != ""
+	confSet := c.ConfDir != ""
+	switch {
+	case homeSet && confSet:
+		err = errors.New("-home must not be used together with -conf")
+	case homeSet:
+		err = locations.SetBaseDir(locations.ConfigBaseDir, c.HomeDir)
+	case confSet:
+		err = locations.SetBaseDir(locations.ConfigBaseDir, c.ConfDir)
+	}
+	if err != nil {
+		return errors.Wrap(err, "Command line options:")
+	}
+	guiCfg := config.GUIConfiguration{
+		RawAddress: c.GUIAddress,
+		APIKey:     c.GUIAPIKey,
+	}
+
+	// Now if the API key and address is not provided (we are not connecting to a remote instance),
+	// try to rip it out of the config.
+	if guiCfg.RawAddress == "" && guiCfg.APIKey == "" {
+		// Load the certs and get the ID
+		cert, err := tls.LoadX509KeyPair(
+			locations.Get(locations.CertFile),
+			locations.Get(locations.KeyFile),
+		)
+		if err != nil {
+			return errors.Wrap(err, "reading device ID")
+		}
+
+		myID := protocol.NewDeviceID(cert.Certificate[0])
+
+		// Load the config
+		cfg, _, err := config.Load(locations.Get(locations.ConfigFile), myID, events.NoopLogger)
+		if err != nil {
+			return errors.Wrap(err, "loading config")
+		}
+
+		guiCfg = cfg.GUI()
+	} else if guiCfg.Address() == "" || guiCfg.APIKey == "" {
+		return errors.New("Both --gui-address and --gui-apikey should be specified")
+	}
+
+	if guiCfg.Address() == "" {
+		return errors.New("Could not find GUI Address")
+	}
+
+	if guiCfg.APIKey == "" {
+		return errors.New("Could not find GUI API key")
+	}
+
+	client := getClient(guiCfg)
+
+	cfg, err := getConfig(client)
+	original := cfg.Copy()
+	if err != nil {
+		return errors.Wrap(err, "getting config")
+	}
+
+	// Copy the config and set the default flags
+	recliCfg := recli.DefaultConfig
+	recliCfg.IDTag.Name = "xml"
+	recliCfg.SkipTag.Name = "json"
+
+	commands, err := recli.New(recliCfg).Construct(&cfg)
+	if err != nil {
+		return errors.Wrap(err, "config reflect")
+	}
+
+	// Implement the same flags at the upper CLI, but do nothing with them.
+	// This is so that the usage text is the same
+	fakeFlags := []cli.Flag{
+		cli.StringFlag{
+			Name:  "gui-address",
+			Value: "URL",
+			Usage: "Override GUI address (e.g. \"http://192.0.2.42:8443\")",
+		},
+		cli.StringFlag{
+			Name:  "gui-apikey",
+			Value: "API-KEY",
+			Usage: "Override GUI API key",
+		},
+		cli.StringFlag{
+			Name:  "home",
+			Value: "PATH",
+			Usage: "Set configuration and data directory",
+		},
+		cli.StringFlag{
+			Name:  "conf",
+			Value: "PATH",
+			Usage: "Set configuration directory (config and keys)",
+		},
+	}
+
+	// Construct the actual CLI
+	app := cli.NewApp()
+	app.Author = "The Syncthing Authors"
+	app.Metadata = map[string]interface{}{
+		"client": client,
+	}
+	app.Commands = []cli.Command{{
+		Name:  "cli",
+		Usage: "Syncthing command line interface",
+		Flags: fakeFlags,
+		Subcommands: []cli.Command{
+			{
+				Name:        "config",
+				HideHelp:    true,
+				Usage:       "Configuration modification command group",
+				Subcommands: commands,
+			},
+			showCommand,
+			operationCommand,
+			errorsCommand,
+		},
+	}}
+
+	tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
+	if !tty {
+		// Not a TTY, consume from stdin
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			input, err := shlex.Split(scanner.Text())
+			if err != nil {
+				return errors.Wrap(err, "parsing input")
+			}
+			if len(input) == 0 {
+				continue
+			}
+			err = app.Run(append(os.Args, input...))
+			if err != nil {
+				return err
+			}
+		}
+		err = scanner.Err()
+		if err != nil {
+			return err
+		}
+	} else {
+		err = app.Run(os.Args)
+		if err != nil {
+			return err
+		}
+	}
+
+	if !reflect.DeepEqual(cfg, original) {
+		body, err := json.MarshalIndent(cfg, "", "  ")
+		if err != nil {
+			return err
+		}
+		resp, err := client.Post("system/config", string(body))
+		if err != nil {
+			return err
+		}
+		if resp.StatusCode != 200 {
+			body, err := responseToBArray(resp)
+			if err != nil {
+				return err
+			}
+			return errors.New(string(body))
+		}
+	}
+	return nil
+}
+
+func parseFlags(c *preCli) error {
+	// kong only needs to parse the global arguments after "cli" and before the
+	// subcommand (if any).
+	if len(os.Args) <= 2 {
+		return nil
+	}
+	args := os.Args[2:]
+	for i := 0; i < len(args); i++ {
+		if !strings.HasPrefix(args[i], "--") {
+			args = args[:i]
+			break
+		}
+		if !strings.Contains(args[i], "=") {
+			i++
+		}
+	}
+	// We don't want kong to print anything nor os.Exit (e.g. on -h)
+	parser, err := kong.New(c, kong.Writers(ioutil.Discard, ioutil.Discard), kong.Exit(func(int) {}))
+	if err != nil {
+		return err
+	}
+	_, err = parser.Parse(args)
+	return err
+}

cmd/syncthing/cli/operations.go

@@ -4,7 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
 	"fmt"
@@ -27,11 +27,6 @@ var operationCommand = cli.Command{
 			Usage:  "Shutdown syncthing",
 			Action: expects(0, emptyPost("system/shutdown")),
 		},
-		{
-			Name:   "reset",
-			Usage:  "Reset syncthing deleting all folders and devices",
-			Action: expects(0, emptyPost("system/reset")),
-		},
 		{
 			Name:   "upgrade",
 			Usage:  "Upgrade syncthing (if a newer version is available)",
@@ -39,7 +34,7 @@ var operationCommand = cli.Command{
 		},
 		{
 			Name:      "folder-override",
-			Usage:     "Override changes on folder (remote for sendonly, local for receiveonly)",
+			Usage:     "Override changes on folder (remote for sendonly, local for receiveonly). WARNING: Destructive - deletes/changes your data.",
 			ArgsUsage: "[folder id]",
 			Action:    expects(1, foldersOverride),
 		},

cmd/syncthing/cli/show.go

@@ -4,7 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
 	"github.com/urfave/cli"
@@ -23,7 +23,7 @@ var showCommand = cli.Command{
 		{
 			Name:   "config-status",
 			Usage:  "Show configuration status, whether or not a restart is required for changes to take effect",
-			Action: expects(0, dumpOutput("system/config/insync")),
+			Action: expects(0, dumpOutput("config/restart-required")),
 		},
 		{
 			Name:   "system",

cmd/syncthing/cli/utils.go

@@ -4,7 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
 	"encoding/json"
@@ -12,7 +12,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"text/tabwriter"
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/urfave/cli"
@@ -45,12 +44,6 @@ func dumpOutput(url string) cli.ActionFunc {
 	}
 }
 
-func newTableWriter() *tabwriter.Writer {
-	writer := new(tabwriter.Writer)
-	writer.Init(os.Stdout, 0, 8, 0, '\t', 0)
-	return writer
-}
-
 func getConfig(c *APIClient) (config.Configuration, error) {
 	cfg := config.Configuration{}
 	response, err := c.Get("system/config")

cmd/syncthing/cpuusage.go

@@ -1,59 +0,0 @@
-// Copyright (C) 2017 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"math"
-	"time"
-
-	metrics "github.com/rcrowley/go-metrics"
-)
-
-const cpuTickRate = 5 * time.Second
-
-type cpuService struct {
-	avg  metrics.EWMA
-	stop chan struct{}
-}
-
-func newCPUService() *cpuService {
-	return &cpuService{
-		// 10 second average. Magic alpha value comes from looking at EWMA package
-		// definitions of EWMA1, EWMA5. The tick rate *must* be five seconds (hard
-		// coded in the EWMA package).
-		avg:  metrics.NewEWMA(1 - math.Exp(-float64(cpuTickRate)/float64(time.Second)/10.0)),
-		stop: make(chan struct{}),
-	}
-}
-
-func (s *cpuService) Serve() {
-	// Initialize prevUsage to an actual value returned by cpuUsage
-	// instead of zero, because at least Windows returns a huge negative
-	// number here that then slowly increments...
-	prevUsage := cpuUsage()
-	ticker := time.NewTicker(cpuTickRate)
-	defer ticker.Stop()
-	for {
-		select {
-		case <-ticker.C:
-			curUsage := cpuUsage()
-			s.avg.Update(int64((curUsage - prevUsage) / time.Millisecond))
-			prevUsage = curUsage
-			s.avg.Tick()
-		case <-s.stop:
-			return
-		}
-	}
-}
-
-func (s *cpuService) Stop() {
-	close(s.stop)
-}
-
-func (s *cpuService) Rate() float64 {
-	return s.avg.Rate()
-}

cmd/syncthing/cpuusage_solaris.go

@@ -1,78 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//+build solaris
-
-package main
-
-import (
-	"encoding/binary"
-	"fmt"
-	"os"
-	"time"
-)
-
-type id_t int32
-type ulong_t uint32
-
-type timestruc_t struct {
-	Tv_sec  int64
-	Tv_nsec int64
-}
-
-func (tv timestruc_t) Nano() int64 {
-	return tv.Tv_sec*1e9 + tv.Tv_nsec
-}
-
-type prusage_t struct {
-	Pr_lwpid    id_t        /* lwp id.  0: process or defunct */
-	Pr_count    int32       /* number of contributing lwps */
-	Pr_tstamp   timestruc_t /* real time stamp, time of read() */
-	Pr_create   timestruc_t /* process/lwp creation time stamp */
-	Pr_term     timestruc_t /* process/lwp termination time stamp */
-	Pr_rtime    timestruc_t /* total lwp real (elapsed) time */
-	Pr_utime    timestruc_t /* user level CPU time */
-	Pr_stime    timestruc_t /* system call CPU time */
-	Pr_ttime    timestruc_t /* other system trap CPU time */
-	Pr_tftime   timestruc_t /* text page fault sleep time */
-	Pr_dftime   timestruc_t /* data page fault sleep time */
-	Pr_kftime   timestruc_t /* kernel page fault sleep time */
-	Pr_ltime    timestruc_t /* user lock wait sleep time */
-	Pr_slptime  timestruc_t /* all other sleep time */
-	Pr_wtime    timestruc_t /* wait-cpu (latency) time */
-	Pr_stoptime timestruc_t /* stopped time */
-	Pr_minf     ulong_t     /* minor page faults */
-	Pr_majf     ulong_t     /* major page faults */
-	Pr_nswap    ulong_t     /* swaps */
-	Pr_inblk    ulong_t     /* input blocks */
-	Pr_oublk    ulong_t     /* output blocks */
-	Pr_msnd     ulong_t     /* messages sent */
-	Pr_mrcv     ulong_t     /* messages received */
-	Pr_sigs     ulong_t     /* signals received */
-	Pr_vctx     ulong_t     /* voluntary context switches */
-	Pr_ictx     ulong_t     /* involuntary context switches */
-	Pr_sysc     ulong_t     /* system calls */
-	Pr_ioch     ulong_t     /* chars read and written */
-
-}
-
-var procFile = fmt.Sprintf("/proc/%d/usage", os.Getpid())
-
-func cpuUsage() time.Duration {
-	fd, err := os.Open(procFile)
-	if err != nil {
-		return 0
-	}
-
-	var rusage prusage_t
-	err = binary.Read(fd, binary.LittleEndian, rusage)
-	fd.Close()
-	if err != nil {
-		return 0
-	}
-
-	return time.Duration(rusage.Pr_utime.Nano() + rusage.Pr_stime.Nano())
-}

cmd/syncthing/cpuusage_windows.go

@@ -1,27 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//+build windows
-
-package main
-
-import "syscall"
-import "time"
-
-func cpuUsage() time.Duration {
-	handle, err := syscall.GetCurrentProcess()
-	if err != nil {
-		return 0
-	}
-	defer syscall.CloseHandle(handle)
-
-	var ctime, etime, ktime, utime syscall.Filetime
-	if err := syscall.GetProcessTimes(handle, &ctime, &etime, &ktime, &utime); err != nil {
-		return 0
-	}
-
-	return time.Duration(ktime.Nanoseconds() + utime.Nanoseconds())
-}

cmd/syncthing/crash_reporting.go

@@ -0,0 +1,152 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/sha256"
+)
+
+const (
+	headRequestTimeout = 10 * time.Second
+	putRequestTimeout  = time.Minute
+)
+
+// uploadPanicLogs attempts to upload all the panic logs in the named
+// directory to the crash reporting server as urlBase. Uploads are attempted
+// with the newest log first.
+//
+// This can can block for a long time. The context can set a final deadline
+// for this.
+func uploadPanicLogs(ctx context.Context, urlBase, dir string) {
+	files, err := filepath.Glob(filepath.Join(dir, "panic-*.log"))
+	if err != nil {
+		l.Warnln("Failed to list panic logs:", err)
+		return
+	}
+
+	sort.Sort(sort.Reverse(sort.StringSlice(files)))
+	for _, file := range files {
+		if strings.Contains(file, ".reported.") {
+			// We've already sent this file. It'll be cleaned out at some
+			// point.
+			continue
+		}
+
+		if err := uploadPanicLog(ctx, urlBase, file); err != nil {
+			l.Warnln("Reporting crash:", err)
+		} else {
+			// Rename the log so we don't have to try to report it again. This
+			// succeeds, or it does not. There is no point complaining about it.
+			_ = os.Rename(file, strings.Replace(file, ".log", ".reported.log", 1))
+		}
+	}
+}
+
+// uploadPanicLog attempts to upload the named panic log to the crash
+// reporting server at urlBase. The panic ID is constructed as the sha256 of
+// the log contents. A HEAD request is made to see if the log has already
+// been reported. If not, a PUT is made with the log contents.
+func uploadPanicLog(ctx context.Context, urlBase, file string) error {
+	data, err := ioutil.ReadFile(file)
+	if err != nil {
+		return err
+	}
+
+	// Remove log lines, for privacy.
+	data = filterLogLines(data)
+
+	hash := fmt.Sprintf("%x", sha256.Sum256(data))
+	l.Infof("Reporting crash found in %s (report ID %s) ...\n", filepath.Base(file), hash[:8])
+
+	url := fmt.Sprintf("%s/%s", urlBase, hash)
+	headReq, err := http.NewRequest(http.MethodHead, url, nil)
+	if err != nil {
+		return err
+	}
+
+	// Set a reasonable timeout on the HEAD request
+	headCtx, headCancel := context.WithTimeout(ctx, headRequestTimeout)
+	defer headCancel()
+	headReq = headReq.WithContext(headCtx)
+
+	resp, err := http.DefaultClient.Do(headReq)
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	if resp.StatusCode == http.StatusOK {
+		// It's known, we're done
+		return nil
+	}
+
+	putReq, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+
+	// Set a reasonable timeout on the PUT request
+	putCtx, putCancel := context.WithTimeout(ctx, putRequestTimeout)
+	defer putCancel()
+	putReq = putReq.WithContext(putCtx)
+
+	resp, err = http.DefaultClient.Do(putReq)
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("upload: %s", resp.Status)
+	}
+
+	return nil
+}
+
+// filterLogLines returns the data without any log lines between the first
+// line and the panic trace. This is done in-place: the original data slice
+// is destroyed.
+func filterLogLines(data []byte) []byte {
+	filtered := data[:0]
+	matched := false
+	for _, line := range bytes.Split(data, []byte("\n")) {
+		switch {
+		case !matched && bytes.HasPrefix(line, []byte("Panic ")):
+			// This begins the panic trace, set the matched flag and append.
+			matched = true
+			fallthrough
+		case len(filtered) == 0 || matched:
+			// This is the first line or inside the panic trace.
+			if len(filtered) > 0 {
+				// We add the newline before rather than after because
+				// bytes.Split sees the \n as *separator* and not line
+				// ender, so ir will generate a last empty line that we
+				// don't really want. (We want to keep blank lines in the
+				// middle of the trace though.)
+				filtered = append(filtered, '\n')
+			}
+			// Remove the device ID prefix. The "plus two" stuff is because
+			// the line will look like "[foo] whatever" and the end variable
+			// will end up pointing at the ] and we want to step over that
+			// and the following space.
+			if end := bytes.Index(line, []byte("]")); end > 1 && end < len(line)-2 && bytes.HasPrefix(line, []byte("[")) {
+				line = line[end+2:]
+			}
+			filtered = append(filtered, line...)
+		}
+	}
+	return filtered
+}

cmd/syncthing/crash_reporting_test.go

@@ -0,0 +1,37 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestFilterLogLines(t *testing.T) {
+	in := []byte(`[ABCD123] syncthing version whatever
+here is more log data
+and more
+...
+and some more
+yet more
+Panic detected at like right now
+here is panic data
+and yet more panic stuff
+`)
+
+	filtered := []byte(`syncthing version whatever
+Panic detected at like right now
+here is panic data
+and yet more panic stuff
+`)
+
+	result := filterLogLines(in)
+	if !bytes.Equal(result, filtered) {
+		t.Logf("%q\n", result)
+		t.Error("it should have been filtered")
+	}
+}

cmd/syncthing/debug.go

@@ -7,16 +7,9 @@
 package main
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
 	l = logger.DefaultLogger.NewFacility("main", "Main package")
 )
-
-func init() {
-	l.SetDebug("main", strings.Contains(os.Getenv("STTRACE"), "main") || os.Getenv("STTRACE") == "all")
-}

cmd/syncthing/decrypt/decrypt.go

@@ -0,0 +1,275 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Package decrypt implements the `syncthing decrypt` subcommand.
+package decrypt
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"path/filepath"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/fs"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/scanner"
+)
+
+type CLI struct {
+	Path       string `arg:"" required:"1" help:"Path to encrypted folder"`
+	To         string `xor:"mode" placeholder:"PATH" help:"Destination directory, when decrypting"`
+	VerifyOnly bool   `xor:"mode" help:"Don't write decrypted files to disk (but verify plaintext hashes)"`
+	Password   string `help:"Folder password for decryption / verification" env:"FOLDER_PASSWORD"`
+	FolderID   string `help:"Folder ID of the encrypted folder, if it cannot be determined automatically"`
+	Continue   bool   `help:"Continue processing next file in case of error, instead of aborting"`
+	Verbose    bool   `help:"Show verbose progress information"`
+	TokenPath  string `placeholder:"PATH" help:"Path to the token file within the folder (used to determine folder ID)"`
+
+	folderKey *[32]byte
+}
+
+type storedEncryptionToken struct {
+	FolderID string
+	Token    []byte
+}
+
+func (c *CLI) Run() error {
+	log.SetFlags(0)
+
+	if c.To == "" && !c.VerifyOnly {
+		return fmt.Errorf("must set --to or --verify")
+	}
+
+	if c.TokenPath == "" {
+		// This is a bit long to show as default in --help
+		c.TokenPath = filepath.Join(config.DefaultMarkerName, config.EncryptionTokenName)
+	}
+
+	if c.FolderID == "" {
+		// We should try to figure out the folder ID
+		folderID, err := c.getFolderID()
+		if err != nil {
+			log.Println("No --folder-id given and couldn't read folder token")
+			return fmt.Errorf("getting folder ID: %w", err)
+		}
+
+		c.FolderID = folderID
+		if c.Verbose {
+			log.Println("Found folder ID:", c.FolderID)
+		}
+	}
+
+	c.folderKey = protocol.KeyFromPassword(c.FolderID, c.Password)
+
+	return c.walk()
+}
+
+// walk finds and processes every file in the encrypted folder
+func (c *CLI) walk() error {
+	srcFs := fs.NewFilesystem(fs.FilesystemTypeBasic, c.Path)
+	var dstFs fs.Filesystem
+	if c.To != "" {
+		dstFs = fs.NewFilesystem(fs.FilesystemTypeBasic, c.To)
+	}
+
+	return srcFs.Walk(".", func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsRegular() {
+			return nil
+		}
+		if fs.IsInternal(path) {
+			return nil
+		}
+
+		return c.withContinue(c.process(srcFs, dstFs, path))
+	})
+}
+
+// If --continue was set we just mention the error and return nil to
+// continue processing.
+func (c *CLI) withContinue(err error) error {
+	if err == nil {
+		return nil
+	}
+	if c.Continue {
+		log.Println("Warning:", err)
+		return nil
+	}
+	return err
+}
+
+// getFolderID returns the folder ID found in the encrypted token, or an
+// error.
+func (c *CLI) getFolderID() (string, error) {
+	tokenPath := filepath.Join(c.Path, c.TokenPath)
+	bs, err := ioutil.ReadFile(tokenPath)
+	if err != nil {
+		return "", fmt.Errorf("reading folder token: %w", err)
+	}
+
+	var tok storedEncryptionToken
+	if err := json.Unmarshal(bs, &tok); err != nil {
+		return "", fmt.Errorf("parsing folder token: %w", err)
+	}
+
+	return tok.FolderID, nil
+}
+
+// process handles the file named path in srcFs, decrypting it into dstFs
+// unless dstFs is nil.
+func (c *CLI) process(srcFs fs.Filesystem, dstFs fs.Filesystem, path string) error {
+	if c.Verbose {
+		log.Printf("Processing %q", path)
+	}
+
+	encFd, err := srcFs.Open(path)
+	if err != nil {
+		return err
+	}
+	defer encFd.Close()
+
+	encFi, err := c.loadEncryptedFileInfo(encFd)
+	if err != nil {
+		return fmt.Errorf("%s: loading metadata trailer: %w", path, err)
+	}
+
+	// Workaround for a bug in <= v1.15.0-rc.5 where we stored names
+	// in native format, while protocol expects wire format (slashes).
+	encFi.Name = osutil.NormalizedFilename(encFi.Name)
+
+	plainFi, err := protocol.DecryptFileInfo(*encFi, c.folderKey)
+	if err != nil {
+		return fmt.Errorf("%s: decrypting metadata: %w", path, err)
+	}
+
+	if c.Verbose {
+		log.Printf("Plaintext filename is %q", plainFi.Name)
+	}
+
+	var plainFd fs.File
+	if dstFs != nil {
+		if err := dstFs.MkdirAll(filepath.Dir(plainFi.Name), 0700); err != nil {
+			return fmt.Errorf("%s: %w", plainFi.Name, err)
+		}
+
+		plainFd, err = dstFs.Create(plainFi.Name)
+		if err != nil {
+			return fmt.Errorf("%s: %w", plainFi.Name, err)
+		}
+		defer plainFd.Close() // also closed explicitly in the return
+	}
+
+	if err := c.decryptFile(encFi, &plainFi, encFd, plainFd); err != nil {
+		// Decrypting the file failed, leaving it in an inconsistent state.
+		// Delete it. Even --continue currently doesn't mean "leave broken
+		// stuff in place", it just means "try the next file instead of
+		// aborting".
+		if plainFd != nil {
+			_ = dstFs.Remove(plainFd.Name())
+		}
+		return fmt.Errorf("%s: %s: %w", path, plainFi.Name, err)
+	} else if c.Verbose {
+		log.Printf("Data verified for %q", plainFi.Name)
+	}
+
+	if plainFd != nil {
+		return plainFd.Close()
+	}
+	return nil
+}
+
+// decryptFile reads, decrypts and verifies all the blocks in src, writing
+// it to dst if dst is non-nil. (If dst is nil it just becomes a
+// read-and-verify operation.)
+func (c *CLI) decryptFile(encFi *protocol.FileInfo, plainFi *protocol.FileInfo, src io.ReaderAt, dst io.WriterAt) error {
+	// The encrypted and plaintext files must consist of an equal number of blocks
+	if len(encFi.Blocks) != len(plainFi.Blocks) {
+		return fmt.Errorf("block count mismatch: encrypted %d != plaintext %d", len(encFi.Blocks), len(plainFi.Blocks))
+	}
+
+	fileKey := protocol.FileKey(plainFi.Name, c.folderKey)
+	for i, encBlock := range encFi.Blocks {
+		// Read the encrypted block
+		buf := make([]byte, encBlock.Size)
+		if _, err := src.ReadAt(buf, encBlock.Offset); err != nil {
+			return fmt.Errorf("encrypted block %d (%d bytes): %w", i, encBlock.Size, err)
+		}
+
+		// Decrypt it
+		dec, err := protocol.DecryptBytes(buf, fileKey)
+		if err != nil {
+			return fmt.Errorf("encrypted block %d (%d bytes): %w", i, encBlock.Size, err)
+		}
+
+		// Verify the block size against the expected plaintext
+		plainBlock := plainFi.Blocks[i]
+		if i == len(plainFi.Blocks)-1 && len(dec) > plainBlock.Size {
+			// The last block might be padded, which is fine (we skip the padding)
+			dec = dec[:plainBlock.Size]
+		} else if len(dec) != plainBlock.Size {
+			return fmt.Errorf("plaintext block %d size mismatch, actual %d != expected %d", i, len(dec), plainBlock.Size)
+		}
+
+		// Verify the hash against the plaintext block info
+		if !scanner.Validate(dec, plainBlock.Hash, 0) {
+			// The block decrypted correctly but fails the hash check. This
+			// is odd and unexpected, but it it's still a valid block from
+			// the source. The file might have changed while we pulled it?
+			err := fmt.Errorf("plaintext block %d (%d bytes) failed validation after decryption", i, plainBlock.Size)
+			if c.Continue {
+				log.Printf("Warning: %s: %s: %v", encFi.Name, plainFi.Name, err)
+			} else {
+				return err
+			}
+		}
+
+		// Write it to the destination, unless we're just verifying.
+		if dst != nil {
+			if _, err := dst.WriteAt(dec, plainBlock.Offset); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// loadEncryptedFileInfo loads the encrypted FileInfo trailer from a file on
+// disk.
+func (c *CLI) loadEncryptedFileInfo(fd fs.File) (*protocol.FileInfo, error) {
+	// Seek to the size of the trailer block
+	if _, err := fd.Seek(-4, io.SeekEnd); err != nil {
+		return nil, err
+	}
+	var bs [4]byte
+	if _, err := io.ReadFull(fd, bs[:]); err != nil {
+		return nil, err
+	}
+	size := int64(binary.BigEndian.Uint32(bs[:]))
+
+	// Seek to the start of the trailer
+	if _, err := fd.Seek(-(4 + size), io.SeekEnd); err != nil {
+		return nil, err
+	}
+	trailer := make([]byte, size)
+	if _, err := io.ReadFull(fd, trailer); err != nil {
+		return nil, err
+	}
+
+	var encFi protocol.FileInfo
+	if err := encFi.Unmarshal(trailer); err != nil {
+		return nil, err
+	}
+
+	return &encFi, nil
+}

cmd/syncthing/heapprof.go

@@ -11,23 +11,20 @@ import (
 	"os"
 	"runtime"
 	"runtime/pprof"
-	"strconv"
 	"syscall"
 	"time"
 )
 
-func init() {
+func startHeapProfiler() {
-	if innerProcess && os.Getenv("STHEAPPROFILE") != "" {
+	l.Debugln("Starting heap profiling")
-		rate := 1
+	go func() {
-		if i, err := strconv.Atoi(os.Getenv("STHEAPPROFILE")); err == nil {
+		err := saveHeapProfiles(1) // Only returns on error
-			rate = i
+		l.Warnln("Heap profiler failed:", err)
-		}
+		panic("Heap profiler failed")
-		l.Debugln("Starting heap profiling")
+	}()
-		go saveHeapProfiles(rate)
-	}
 }
 
-func saveHeapProfiles(rate int) {
+func saveHeapProfiles(rate int) error {
 	runtime.MemProfileRate = rate
 	var memstats, prevMemstats runtime.MemStats
 
@@ -38,21 +35,21 @@ func saveHeapProfiles(rate int) {
 		if memstats.HeapInuse > prevMemstats.HeapInuse {
 			fd, err := os.Create(name + ".tmp")
 			if err != nil {
-				panic(err)
+				return err
 			}
 			err = pprof.WriteHeapProfile(fd)
 			if err != nil {
-				panic(err)
+				return err
 			}
 			err = fd.Close()
 			if err != nil {
-				panic(err)
+				return err
 			}
 
 			os.Remove(name) // Error deliberately ignored
 			err = os.Rename(name+".tmp", name)
 			if err != nil {
-				panic(err)
+				return err
 			}
 
 			prevMemstats = memstats

cmd/syncthing/main_test.go

@@ -1,38 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"testing"
-
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-func TestShortIDCheck(t *testing.T) {
-	cfg := config.Wrap("/tmp/test", config.Configuration{
-		Devices: []config.DeviceConfiguration{
-			{DeviceID: protocol.DeviceID{8, 16, 24, 32, 40, 48, 56, 0, 0}},
-			{DeviceID: protocol.DeviceID{8, 16, 24, 32, 40, 48, 56, 1, 1}}, // first 56 bits same, differ in the first 64 bits
-		},
-	})
-
-	if err := checkShortIDs(cfg); err != nil {
-		t.Error("Unexpected error:", err)
-	}
-
-	cfg = config.Wrap("/tmp/test", config.Configuration{
-		Devices: []config.DeviceConfiguration{
-			{DeviceID: protocol.DeviceID{8, 16, 24, 32, 40, 48, 56, 64, 0}},
-			{DeviceID: protocol.DeviceID{8, 16, 24, 32, 40, 48, 56, 64, 1}}, // first 64 bits same
-		},
-	})
-
-	if err := checkShortIDs(cfg); err == nil {
-		t.Error("Should have gotten an error")
-	}
-}

cmd/syncthing/monitor.go

@@ -8,17 +8,24 @@ package main
 
 import (
 	"bufio"
+	"context"
+	"fmt"
 	"io"
 	"os"
 	"os/exec"
 	"os/signal"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/svcutil"
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
@@ -29,41 +36,58 @@ var (
 )
 
 const (
-	countRestarts         = 4
+	restartCounts         = 4
-	loopThreshold         = 60 * time.Second
+	restartPause          = 1 * time.Second
+	restartLoopThreshold  = 60 * time.Second
 	logFileAutoCloseDelay = 5 * time.Second
 	logFileMaxOpenTime    = time.Minute
+	panicUploadMaxWait    = 30 * time.Second
+	panicUploadNoticeWait = 10 * time.Second
 )
 
-func monitorMain(runtimeOptions RuntimeOptions) {
+func monitorMain(options serveOptions) {
 	l.SetPrefix("[monitor] ")
 
 	var dst io.Writer = os.Stdout
 
-	logFile := runtimeOptions.logFile
+	logFile := options.LogFile
 	if logFile != "-" {
-		var fileDst io.Writer = newAutoclosedFile(logFile, logFileAutoCloseDelay, logFileMaxOpenTime)
+		if expanded, err := fs.ExpandTilde(logFile); err == nil {
-
+			logFile = expanded
-		if runtime.GOOS == "windows" {
-			// Translate line breaks to Windows standard
-			fileDst = osutil.ReplacingWriter{
-				Writer: fileDst,
-				From:   '\n',
-				To:     []byte{'\r', '\n'},
-			}
 		}
+		var fileDst io.Writer
+		var err error
+		open := func(name string) (io.WriteCloser, error) {
+			return newAutoclosedFile(name, logFileAutoCloseDelay, logFileMaxOpenTime)
+		}
+		if options.LogMaxSize > 0 {
+			fileDst, err = newRotatedFile(logFile, open, int64(options.LogMaxSize), options.LogMaxFiles)
+		} else {
+			fileDst, err = open(logFile)
+		}
+		if err != nil {
+			l.Warnln("Failed to setup logging to file, proceeding with logging to stdout only:", err)
+		} else {
+			if runtime.GOOS == "windows" {
+				// Translate line breaks to Windows standard
+				fileDst = osutil.ReplacingWriter{
+					Writer: fileDst,
+					From:   '\n',
+					To:     []byte{'\r', '\n'},
+				}
+			}
 
-		// Log to both stdout and file.
+			// Log to both stdout and file.
-		dst = io.MultiWriter(dst, fileDst)
+			dst = io.MultiWriter(dst, fileDst)
 
-		l.Infof(`Log output saved to file "%s"`, logFile)
+			l.Infof(`Log output saved to file "%s"`, logFile)
+		}
 	}
 
 	args := os.Args
-	var restarts [countRestarts]time.Time
+	var restarts [restartCounts]time.Time
 
 	stopSign := make(chan os.Signal, 1)
-	sigTerm := syscall.Signal(15)
 	signal.Notify(stopSign, os.Interrupt, sigTerm)
 	restartSign := make(chan os.Signal, 1)
 	sigHup := syscall.Signal(1)
@@ -72,9 +96,11 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 	childEnv := childEnv()
 	first := true
 	for {
-		if t := time.Since(restarts[0]); t < loopThreshold {
+		maybeReportPanics()
-			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
+
-			os.Exit(exitError)
+		if t := time.Since(restarts[0]); t < restartLoopThreshold {
+			l.Warnf("%d restarts in %v; not retrying further", restartCounts, t)
+			os.Exit(svcutil.ExitError.AsInt())
 		}
 
 		copy(restarts[0:], restarts[1:])
@@ -93,10 +119,11 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 			panic(err)
 		}
 
-		l.Infoln("Starting syncthing")
+		l.Debugln("Starting syncthing")
 		err = cmd.Start()
 		if err != nil {
-			panic(err)
+			l.Warnln("Error starting the main Syncthing process:", err)
+			panic("Error starting the main Syncthing process")
 		}
 
 		stdoutMut.Lock()
@@ -125,12 +152,13 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 			exit <- cmd.Wait()
 		}()
 
+		stopped := false
 		select {
 		case s := <-stopSign:
 			l.Infof("Signal %d received; exiting", s)
 			cmd.Process.Signal(sigTerm)
-			<-exit
+			err = <-exit
-			return
+			stopped = true
 
 		case s := <-restartSign:
 			l.Infof("Signal %d received; restarting", s)
@@ -138,27 +166,35 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 			err = <-exit
 
 		case err = <-exit:
-			if err == nil {
+		}
-				// Successful exit indicates an intentional shutdown
+
-				return
+		if err == nil {
-			} else if exiterr, ok := err.(*exec.ExitError); ok {
+			// Successful exit indicates an intentional shutdown
-				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+			os.Exit(svcutil.ExitSuccess.AsInt())
-					switch status.ExitStatus() {
+		}
-					case exitUpgrading:
+
-						// Restart the monitor process to release the .old
+		if exiterr, ok := err.(*exec.ExitError); ok {
-						// binary as part of the upgrade process.
+			exitCode := exiterr.ExitCode()
-						l.Infoln("Restarting monitor...")
+			if stopped || options.NoRestart {
-						if err = restartMonitor(args); err != nil {
+				os.Exit(exitCode)
-							l.Warnln("Restart:", err)
+			}
-						}
+			if exitCode == svcutil.ExitUpgrade.AsInt() {
-						return
+				// Restart the monitor process to release the .old
-					}
+				// binary as part of the upgrade process.
+				l.Infoln("Restarting monitor...")
+				if err = restartMonitor(args); err != nil {
+					l.Warnln("Restart:", err)
 				}
+				os.Exit(exitCode)
 			}
 		}
 
+		if options.NoRestart {
+			os.Exit(svcutil.ExitError.AsInt())
+		}
+
 		l.Infoln("Syncthing exited:", err)
-		time.Sleep(1 * time.Second)
+		time.Sleep(restartPause)
 
 		if first {
 			// Let the next child process know that this is not the first time
@@ -173,6 +209,13 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
 	br := bufio.NewReader(stderr)
 
 	var panicFd *os.File
+	defer func() {
+		if panicFd != nil {
+			_ = panicFd.Close()
+			maybeReportPanics()
+		}
+	}()
+
 	for {
 		line, err := br.ReadString('\n')
 		if err != nil {
@@ -269,6 +312,11 @@ func copyStdout(stdout io.Reader, dst io.Writer) {
 }
 
 func restartMonitor(args []string) error {
+	// Set the STRESTART environment variable to indicate to the next
+	// process that this is a restart and not initial start. This prevents
+	// opening the browser on startup.
+	os.Setenv("STRESTART", "yes")
+
 	if runtime.GOOS != "windows" {
 		// syscall.Exec is the cleanest way to restart on Unixes as it
 		// replaces the current process with the new one, keeping the pid and
@@ -304,6 +352,89 @@ func restartMonitorWindows(args []string) error {
 	return cmd.Start()
 }
 
+// rotatedFile keeps a set of rotating logs. There will be the base file plus up
+// to maxFiles rotated ones, each ~ maxSize bytes large.
+type rotatedFile struct {
+	name        string
+	create      createFn
+	maxSize     int64 // bytes
+	maxFiles    int
+	currentFile io.WriteCloser
+	currentSize int64
+}
+
+type createFn func(name string) (io.WriteCloser, error)
+
+func newRotatedFile(name string, create createFn, maxSize int64, maxFiles int) (*rotatedFile, error) {
+	var size int64
+	if info, err := os.Lstat(name); err != nil {
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+		size = 0
+	} else {
+		size = info.Size()
+	}
+	writer, err := create(name)
+	if err != nil {
+		return nil, err
+	}
+	return &rotatedFile{
+		name:        name,
+		create:      create,
+		maxSize:     maxSize,
+		maxFiles:    maxFiles,
+		currentFile: writer,
+		currentSize: size,
+	}, nil
+}
+
+func (r *rotatedFile) Write(bs []byte) (int, error) {
+	// Check if we're about to exceed the max size, and if so close this
+	// file so we'll start on a new one.
+	if r.currentSize+int64(len(bs)) > r.maxSize {
+		r.currentFile.Close()
+		r.currentSize = 0
+		r.rotate()
+		f, err := r.create(r.name)
+		if err != nil {
+			return 0, err
+		}
+		r.currentFile = f
+	}
+
+	n, err := r.currentFile.Write(bs)
+	r.currentSize += int64(n)
+	return n, err
+}
+
+func (r *rotatedFile) rotate() {
+	// The files are named "name", "name.0", "name.1", ...
+	// "name.(r.maxFiles-1)". Increase the numbers on the
+	// suffixed ones.
+	for i := r.maxFiles - 1; i > 0; i-- {
+		from := numberedFile(r.name, i-1)
+		to := numberedFile(r.name, i)
+		err := os.Rename(from, to)
+		if err != nil && !os.IsNotExist(err) {
+			fmt.Println("LOG: Rotating logs:", err)
+		}
+	}
+
+	// Rename the base to base.0
+	err := os.Rename(r.name, numberedFile(r.name, 0))
+	if err != nil && !os.IsNotExist(err) {
+		fmt.Println("LOG: Rotating logs:", err)
+	}
+}
+
+// numberedFile adds the number between the file name and the extension.
+func numberedFile(name string, num int) string {
+	ext := filepath.Ext(name) // contains the dot
+	withoutExt := name[:len(name)-len(ext)]
+	return fmt.Sprintf("%s.%d%s", withoutExt, num, ext)
+}
+
 // An autoclosedFile is an io.WriteCloser that opens itself for appending on
 // Write() and closes itself after an interval of no writes (closeDelay) or
 // when the file has been open for too long (maxOpenTime). A call to Write()
@@ -322,7 +453,7 @@ type autoclosedFile struct {
 	mut sync.Mutex
 }
 
-func newAutoclosedFile(name string, closeDelay, maxOpenTime time.Duration) *autoclosedFile {
+func newAutoclosedFile(name string, closeDelay, maxOpenTime time.Duration) (*autoclosedFile, error) {
 	f := &autoclosedFile{
 		name:        name,
 		closeDelay:  closeDelay,
@@ -331,8 +462,13 @@ func newAutoclosedFile(name string, closeDelay, maxOpenTime time.Duration) *auto
 		closed:      make(chan struct{}),
 		closeTimer:  time.NewTimer(time.Minute),
 	}
+	f.mut.Lock()
+	defer f.mut.Unlock()
+	if err := f.ensureOpenLocked(); err != nil {
+		return nil, err
+	}
 	go f.closerLoop()
-	return f
+	return f, nil
 }
 
 func (f *autoclosedFile) Write(bs []byte) (int, error) {
@@ -340,7 +476,7 @@ func (f *autoclosedFile) Write(bs []byte) (int, error) {
 	defer f.mut.Unlock()
 
 	// Make sure the file is open for appending
-	if err := f.ensureOpen(); err != nil {
+	if err := f.ensureOpenLocked(); err != nil {
 		return 0, err
 	}
 
@@ -370,22 +506,14 @@ func (f *autoclosedFile) Close() error {
 }
 
 // Must be called with f.mut held!
-func (f *autoclosedFile) ensureOpen() error {
+func (f *autoclosedFile) ensureOpenLocked() error {
 	if f.fd != nil {
 		// File is already open
 		return nil
 	}
 
 	// We open the file for write only, and create it if it doesn't exist.
-	flags := os.O_WRONLY | os.O_CREATE
+	flags := os.O_WRONLY | os.O_CREATE | os.O_APPEND
-	if f.opened.IsZero() {
-		// This is the first time we are opening the file. We should truncate
-		// it to better emulate an os.Create() call.
-		flags |= os.O_TRUNC
-	} else {
-		// The file was already opened once, so we should append to it.
-		flags |= os.O_APPEND
-	}
 
 	fd, err := os.OpenFile(f.name, flags, 0644)
 	if err != nil {
@@ -430,3 +558,39 @@ func childEnv() []string {
 	env = append(env, "STMONITORED=yes")
 	return env
 }
+
+// maybeReportPanics tries to figure out if crash reporting is on or off,
+// and reports any panics it can find if it's enabled. We spend at most
+// panicUploadMaxWait uploading panics...
+func maybeReportPanics() {
+	// Try to get a config to see if/where panics should be reported.
+	cfg, err := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger, true)
+	if err != nil {
+		l.Warnln("Couldn't load config; not reporting crash")
+		return
+	}
+
+	// Bail if we're not supposed to report panics.
+	opts := cfg.Options()
+	if !opts.CREnabled {
+		return
+	}
+
+	// Set up a timeout on the whole operation.
+	ctx, cancel := context.WithTimeout(context.Background(), panicUploadMaxWait)
+	defer cancel()
+
+	// Print a notice if the upload takes a long time.
+	go func() {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(panicUploadNoticeWait):
+			l.Warnln("Uploading crash reports is taking a while, please wait...")
+		}
+	}()
+
+	// Report the panics.
+	dir := locations.GetBaseDir(locations.ConfigBaseDir)
+	uploadPanicLogs(ctx, opts.CRURL, dir)
+}

cmd/syncthing/monitor_test.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -14,6 +15,126 @@ import (
 	"time"
 )
 
+func TestRotatedFile(t *testing.T) {
+	// Verify that log rotation happens.
+
+	dir, err := ioutil.TempDir("", "syncthing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	open := func(name string) (io.WriteCloser, error) {
+		return os.Create(name)
+	}
+
+	logName := filepath.Join(dir, "log.txt")
+	testData := []byte("12345678\n")
+	maxSize := int64(len(testData) + len(testData)/2)
+
+	// We allow the log file plus two rotated copies.
+	rf, err := newRotatedFile(logName, open, maxSize, 2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Write some bytes.
+	if _, err := rf.Write(testData); err != nil {
+		t.Fatal(err)
+	}
+	// They should be in the log.
+	checkSize(t, logName, len(testData))
+	checkNotExist(t, logName+".0")
+
+	// Write some more bytes. We should rotate and write into a new file as the
+	// new bytes don't fit.
+	if _, err := rf.Write(testData); err != nil {
+		t.Fatal(err)
+	}
+	checkSize(t, logName, len(testData))
+	checkSize(t, numberedFile(logName, 0), len(testData))
+	checkNotExist(t, logName+".1")
+
+	// Write another byte. That should fit without causing an extra rotate.
+	_, _ = rf.Write([]byte{42})
+	checkSize(t, logName, len(testData)+1)
+	checkSize(t, numberedFile(logName, 0), len(testData))
+	checkNotExist(t, numberedFile(logName, 1))
+
+	// Write some more bytes. We should rotate and write into a new file as the
+	// new bytes don't fit.
+	if _, err := rf.Write(testData); err != nil {
+		t.Fatal(err)
+	}
+	checkSize(t, logName, len(testData))
+	checkSize(t, numberedFile(logName, 0), len(testData)+1) // the one we wrote extra to, now rotated
+	checkSize(t, numberedFile(logName, 1), len(testData))
+	checkNotExist(t, numberedFile(logName, 2))
+
+	// Write some more bytes. We should rotate and write into a new file as the
+	// new bytes don't fit.
+	if _, err := rf.Write(testData); err != nil {
+		t.Fatal(err)
+	}
+	checkSize(t, logName, len(testData))
+	checkSize(t, numberedFile(logName, 0), len(testData))
+	checkSize(t, numberedFile(logName, 1), len(testData)+1)
+	checkNotExist(t, numberedFile(logName, 2)) // exceeds maxFiles so deleted
+}
+
+func TestNumberedFile(t *testing.T) {
+	// Mostly just illustrates where the number ends up and makes sure it
+	// doesn't crash without an extension.
+
+	cases := []struct {
+		in  string
+		num int
+		out string
+	}{
+		{
+			in:  "syncthing.log",
+			num: 42,
+			out: "syncthing.42.log",
+		},
+		{
+			in:  filepath.Join("asdfasdf", "syncthing.log.txt"),
+			num: 42,
+			out: filepath.Join("asdfasdf", "syncthing.log.42.txt"),
+		},
+		{
+			in:  "syncthing-log",
+			num: 42,
+			out: "syncthing-log.42",
+		},
+	}
+
+	for _, tc := range cases {
+		res := numberedFile(tc.in, tc.num)
+		if res != tc.out {
+			t.Errorf("numberedFile(%q, %d) => %q, expected %q", tc.in, tc.num, res, tc.out)
+		}
+	}
+}
+
+func checkSize(t *testing.T, name string, size int) {
+	t.Helper()
+	info, err := os.Lstat(name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.Size() != int64(size) {
+		t.Errorf("%s wrong size: %d != expected %d", name, info.Size(), size)
+	}
+}
+
+func checkNotExist(t *testing.T, name string) {
+	t.Helper()
+	_, err := os.Lstat(name)
+	if !os.IsNotExist(err) {
+		t.Errorf("%s should not exist", name)
+	}
+}
+
 func TestAutoClosedFile(t *testing.T) {
 	os.RemoveAll("_autoclose")
 	defer os.RemoveAll("_autoclose")
@@ -22,7 +143,10 @@ func TestAutoClosedFile(t *testing.T) {
 	data := []byte("hello, world\n")
 
 	// An autoclosed file that closes very quickly
-	ac := newAutoclosedFile(file, time.Millisecond, time.Millisecond)
+	ac, err := newAutoclosedFile(file, time.Millisecond, time.Millisecond)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	// Write some data.
 	if _, err := ac.Write(data); err != nil {
@@ -64,21 +188,23 @@ func TestAutoClosedFile(t *testing.T) {
 	}
 
 	// Open the file again.
-	ac = newAutoclosedFile(file, time.Second, time.Second)
+	ac, err = newAutoclosedFile(file, time.Second, time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	// Write something
 	if _, err := ac.Write(data); err != nil {
 		t.Fatal(err)
 	}
 
-	// It should now contain only one write, because the first open
+	// It should now contain three writes, as the file is always opened for appending
-	// should be a truncate.
 	bs, err = ioutil.ReadFile(file)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(bs) != len(data) {
+	if len(bs) != 3*len(data) {
-		t.Fatalf("Write failed, expected %d bytes, not %d", len(data), len(bs))
+		t.Fatalf("Write failed, expected %d bytes, not %d", 3*len(data), len(bs))
 	}
 
 	// Close.

cmd/syncthing/options_others.go

@@ -1,13 +1,13 @@
-// Copyright (C) 2017 The Syncthing Authors.
+// Copyright (C) 2021 The Syncthing Authors.
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package api
+// +build !windows
 
-type mockedCPUService struct{}
+package main
 
-func (*mockedCPUService) Rate() float64 {
+type buildServeOptions struct {
-	return 42
+	HideConsole bool `hidden:""`
 }

cmd/syncthing/options_windows.go

@@ -0,0 +1,11 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+type buildServeOptions struct {
+	HideConsole bool `name:"no-console" help:"Hide console window"`
+}

cmd/syncthing/perfstats_unix.go

@@ -18,10 +18,8 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func init() {
+func startPerfStats() {
-	if innerProcess && os.Getenv("STPERFSTATS") != "" {
+	go savePerfStats(fmt.Sprintf("perfstats-%d.csv", syscall.Getpid()))
-		go savePerfStats(fmt.Sprintf("perfstats-%d.csv", syscall.Getpid()))
-	}
 }
 
 func savePerfStats(file string) {

cmd/syncthing/perfstats_unsupported.go

@@ -1,13 +1,12 @@
-// Copyright (C) 2017 The Syncthing Authors.
+// Copyright (C) 2021 The Syncthing Authors.
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//+build noupgrade
+// +build solaris windows
 
-package build
+package main
 
-func init() {
+func startPerfStats() {
-	Tags = append(Tags, "noupgrade")
 }

cmd/syncthing/usage.go

@@ -1,57 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"flag"
-	"fmt"
-	"io"
-	"text/tabwriter"
-)
-
-func optionTable(w io.Writer, rows [][]string) {
-	tw := tabwriter.NewWriter(w, 2, 4, 2, ' ', 0)
-	for _, row := range rows {
-		for i, cell := range row {
-			if i > 0 {
-				tw.Write([]byte("\t"))
-			}
-			tw.Write([]byte(cell))
-		}
-		tw.Write([]byte("\n"))
-	}
-	tw.Flush()
-}
-
-func usageFor(fs *flag.FlagSet, usage string, extra string) func() {
-	return func() {
-		var b bytes.Buffer
-		b.WriteString("Usage:\n  " + usage + "\n")
-
-		var options [][]string
-		fs.VisitAll(func(f *flag.Flag) {
-			var opt = "  -" + f.Name
-
-			if f.DefValue != "false" {
-				opt += "=" + fmt.Sprintf(`"%s"`, f.DefValue)
-			}
-			options = append(options, []string{opt, f.Usage})
-		})
-
-		if len(options) > 0 {
-			b.WriteString("\nOptions:\n")
-			optionTable(&b, options)
-		}
-
-		fmt.Println(b.String())
-
-		if len(extra) > 0 {
-			fmt.Println(extra)
-		}
-	}
-}