docker: Fix handling of PCAP variable (fixes #8567) (#8568)

This correctly handles the absense of the PCAP environment variable,
which was broken in the previous change.

.codecov.yml

@@ -0,0 +1,20 @@
+comment: false
+
+coverage:
+  range: "40...100"
+  precision: 1
+  status:
+    patch:
+      default:
+        informational: true
+    project:
+      default:
+        informational: true
+
+github_checks:
+  annotations: false
+
+ignore:
+  - "**.pb.go"
+  - "**_mocked.go"
+  - "**/mocks/*"

.deepsource.toml

@@ -0,0 +1,12 @@
+version = 1
+
+exclude_patterns = ["**/*.pb.go"]
+test_patterns = ["**/*_test.go"]
+
+[[analyzers]]
+name = "go"
+enabled = true
+
+  [analyzers.meta]
+  import_paths = ["github.com/syncthing/syncthing"]
+  build_tags = ["noassets"]

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+/AUTHORS    @calmh
+/*.md       @calmh

.github/FUNDING.yml

@@ -0,0 +1,11 @@
+github: syncthing
+custom: "https://syncthing.net/donations/"
+
+# patreon: # Replace with a single Patreon username
+# open_collective: # Replace with a single Open Collective username
+# ko_fi: # Replace with a single Ko-fi username
+# tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+# community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+# liberapay: # Replace with a single Liberapay username
+# issuehunt: # Replace with a single IssueHunt username
+# otechie: # Replace with a single Otechie username

.github/ISSUE_TEMPLATE/01-feature.md

@@ -0,0 +1,13 @@
+---
+name: Feature request
+about: If you're just not sure how to do something, see "ask a question".
+labels: enhancement, needs-triage
+---
+
+### Include required information
+
+Please be sure to include at least:
+
+ - what problem your new feature would solve
+ - how or why you think it is generally useful (i.e., not just for you)
+ - what alternatives or workarounds you considered

.github/ISSUE_TEMPLATE/02-bug.md

@@ -0,0 +1,23 @@
+---
+name: Bug report
+about: If you're actually looking for support, see "ask a question".
+labels: bug, needs-triage
+---
+
+### Does your log mention database corruption?
+
+If your Syncthing log reports panics because of database corruption it is
+most likely a fault with your system's storage or memory. Affected log
+entries will contain lines starting with `panic: leveldb`. You will need to
+delete the index database to clear this, by running `syncthing
+-reset-database`.
+
+### Include required information
+
+Please be sure to include at least:
+
+ - which version of Syncthing and what operating system you are using
+ - browser and version, if applicable
+ - what happened,
+ - what you expected to happen instead, and
+ - any steps to reproduce the problem.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: I need help / I have a question
+    url: https://forum.syncthing.net/
+    about: Ask questions, get support, and discuss with other community members.
+  - name: Android issues
+    url: https://github.com/syncthing/syncthing-android/issues/
+    about: The Android app has its own issue tracker.

.github/PULL_REQUEST_TEMPLATE.md

@@ -20,13 +20,8 @@ If this is a user visible change (including API and protocol changes), add a lin
 to the corresponding pull request on https://github.com/syncthing/docs or describe
 the documentation changes necessary.
 
-### Authorship
+## Authorship
+
+Your name and email will be added automatically to the AUTHORS file
+based on the commit metadata.
 
-Every author of a code contribution (Go, Javascript, HTML, CSS etc, with the
-possible exception of minor typo corrections and similar) is recorded in the
-AUTHORS and NICKS files and the in-GUI credits. If this is your first
-contribution, a maintainer will add you properly before accepting the
-contribution. You need not do so yourself or worry about the fact that the
-"authors" automated test fails. However, if your name (such as you want it
-presented in the credits) is not visible on your Github profile or in your
-commit messages, please assist by providing it here.

.github/SECURITY.md

@@ -0,0 +1,10 @@
+## Reporting a Vulnerability
+
+If you believe that you've found a Syncthing-related security vulnerability,
+please report it by sending email to the address security@syncthing.net. The
+[PGP key for security@syncthing.net
+(B683AD7B76CAB013)](https://syncthing.net/security-key.txt) can be used to
+send encrypted mail or to verify responses received from that address.
+
+You can read more about Syncthing security at
+https://syncthing.net/security/.

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: weekly
+  open-pull-requests-limit: 10
+  
+- package-ecosystem: "gomod"
+  directory: "/"
+  schedule:
+    interval: weekly
+  open-pull-requests-limit: 10

.github/workflows/update-docs-translations.yaml

@@ -0,0 +1,29 @@
+name: Update translations and documentation
+on:
+  workflow_dispatch:
+  schedule:
+    - cron:  '42 3 * * 1'
+
+jobs:
+
+  update_transifex_docs:
+    runs-on: ubuntu-latest
+    name: Update translations and documentation
+    steps:
+      - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
+      - uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # v2
+        with:
+          go-version: ^1.18.4
+      - run: |
+          set -euo pipefail
+          git config --global user.name 'Syncthing Release Automation'
+          git config --global user.email 'release@syncthing.net'
+          bash build.sh translate
+          bash build.sh prerelease
+          git push
+        env:
+          TRANSIFEX_USER: ${{ secrets.TRANSIFEX_USER }}
+          TRANSIFEX_PASS: ${{ secrets.TRANSIFEX_PASS }}

.gitignore

@@ -15,5 +15,8 @@ coverage.xml
 syncthing.sig
 RELEASE
 deb
-lib/auto/gui.files.go
-snapcraft.yaml
+*.bz2
+/repos
+/proto/scripts/protoc-gen-gosyncthing
+/gui/next-gen-gui
+.idea

.golangci.yml

@@ -0,0 +1,26 @@
+linters-settings:
+  maligned:
+    suggest-new: true
+
+linters:
+  enable-all: true
+  disable:
+    - goimports
+    - depguard
+    - lll
+    - gochecknoinits
+    - gochecknoglobals
+    - gofmt
+    - scopelint
+    - gocyclo
+    - funlen
+    - wsl
+    - gocognit
+    - godox
+
+service:
+  golangci-lint-version: 1.21.x
+  prepare:
+    - rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
+    - GO111MODULE=on go mod vendor
+    - go run build.go assets

AUTHORS

@@ -1,126 +1,304 @@
 # This is the official list of Syncthing authors for copyright purposes.
-# The format is:
+#
+# THIS FILE IS MOSTLY AUTO GENERATED. IF YOU'VE MADE A COMMIT TO THE
+# REPOSITORY YOU WILL BE ADDED HERE AUTOMATICALLY WITHOUT THE NEED FOR
+# ANY MANUAL ACTION.
+#
+# That said, you are welcome to correct your name or add a nickname / GitHub
+# user name as appropriate. The format is:
 #
 #    Name Name Name (nickname) <email1@example.com> <email2@example.com>
 #
-# After changing this list, run "go run script/authors.go" to sort and update
-# the GUI HTML.
+# The in-GUI authors list is periodically automatically updated from the
+# contents of this file.
 #
 
 Aaron Bieber (qbit) <qbit@deftly.net>
-Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com>
+Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com> <adam@proactiveservices.co.uk>
 Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
+Alan Pope <alan@popey.com>
+Alberto Donato <albertodonato@users.noreply.github.com>
 Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
+Alex Lindeman <139387+aelindeman@users.noreply.github.com>
+Alex Xu <alex.hello71@gmail.com>
 Alexander Graf (alex2108) <register-github@alex-graf.de>
 Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
+Aman Gupta <aman@tmm1.net>
 Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
+andresvia <andres.via@gmail.com>
 Andrew Dunham (andrew-d) <andrew@du.nham.ca>
-Andrey D (scienmind) <scintertech@cryptolab.net>
+Andrew Meyer <andrewm.bpi@gmail.com>
+Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
+Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
+André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
+andyleap <andyleap@gmail.com>
+Anjan Momi <anjan@momi.ca>
 Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
 Antony Male (canton7) <antony.male@gmail.com>
+Anur <anurnomeru@163.com>
+Aranjedeath <Aranjedeath@users.noreply.github.com>
+Arkadiusz Tymiński <gevleeog@gmail.com>
+Aroun <login@b-vo.fr>
 Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
-Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com>
+Artur Zubilewicz <AkaZecik@users.noreply.github.com>
+Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
+Aurélien Rainone <476650+arl@users.noreply.github.com>
+BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
 Bart De Vries (mogwa1) <devriesb@gmail.com>
 Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
 Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
 Ben Shepherd (benshep) <bjashepherd@gmail.com>
 Ben Sidhom (bsidhom) <bsidhom@gmail.com>
+Benedikt Heine (bebehei) <bebe@bebehei.de>
+Benedikt Morbach <benedikt.morbach@googlemail.com>
+Benjamin Nater <17193640+bn4t@users.noreply.github.com>
+Benno Fünfstück <benno.fuenfstueck@gmail.com>
 Benny Ng (tpng) <benny.tpng@gmail.com>
+boomsquared <54829195+boomsquared@users.noreply.github.com>
+Boqin Qin <bobbqqin@bupt.edu.cn>
+Boris Rybalkin <ribalkin@gmail.com>
 Brandon Philips (philips) <brandon@ifup.org>
 Brendan Long (brendanlong) <self@brendanlong.com>
 Brian R. Becker (brbecker) <brbecker@gmail.com>
+bt90 <btom1990@googlemail.com>
 Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
-Carsten Hagemann (Moter8) <moter8@gmail.com>
-Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
+Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
+Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
 Cedric Staniewski (xduugu) <cedric@gmx.ca>
+chenrui <rui@meetup.com>
+Chih-Hsuan Yen <yan12125@gmail.com>
+Choongkyu <choongkyu.kim+gh@gmail.com> <vapidlyrapid+gh@gmail.com>
 Chris Howie (cdhowie) <me@chrishowie.com>
 Chris Joel (cdata) <chris@scriptolo.gy>
+Chris Tonkinson <chris@masterbran.ch>
+Christian Prescott <me@christianprescott.com>
+chucic <chucic@seznam.cz>
 Colin Kennedy (moshen) <moshen.colin@gmail.com>
+Cromefire_ <tim.l@nghorst.net> <26320625+cromefire@users.noreply.github.com>
+cui fliter <imcusg@gmail.com>
+Cyprien Devillez <cypx@users.noreply.github.com>
+Dale Visser <dale.visser@live.com>
+Dan <benda.daniel@gmail.com>
+Daniel Barczyk <46358936+DanielBarczyk@users.noreply.github.com>
 Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
 Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
 Daniel Martí (mvdan) <mvdan@mvdan.cc>
 Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
 David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
+deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com>
 Denis A. (dva) <denisva@gmail.com>
 Dennis Wilson (snnd) <dw@risu.io>
+dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com> <27856297+dependabot-preview[bot]@users.noreply.github.com>
+dependabot[bot] <dependabot[bot]@users.noreply.github.com> <49699333+dependabot[bot]@users.noreply.github.com>
+derekriemer <derek.riemer@colorado.edu>
+desbma <desbma@users.noreply.github.com>
+Devon G. Redekopp <devon@redekopp.com>
 Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
+Domenic Horner <domenic@tgxn.net>
 Dominik Heidler (asdil12) <dominik@heidler.eu>
 Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
+Elliot Huffman <thelich2@gmail.com>
 Emil Hessman (ceh) <emil@hessman.se>
+Eng Zer Jun <engzerjun@gmail.com>
+Eric Lesiuta <elesiuta@gmail.com>
+Eric P <eric@kastelo.net>
 Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
+Evan Spensley <94762716+0evan@users.noreply.github.com>
+Evgeny Kuznetsov <evgeny@kuznetsov.md>
 Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
 Felix Ableitner (Nutomic) <me@nutomic.com>
+Felix Lampe <mail@flampe.de>
 Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
 Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
 Frank Isemann (fti7) <frank@isemann.name>
+Gahl Saraf <saraf.gahl@gmail.com> <gahl@raftt.io>
+georgespatton <georgespatton@users.noreply.github.com>
+ghjklw <malo@jaffre.info>
 Gilli Sigurdsson (gillisig) <gilli@vx.is>
+Gleb Sinyavskiy <zhulik.gleb@gmail.com>
+Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
+greatroar <61184462+greatroar@users.noreply.github.com>
+Greg <gco@jazzhaiku.com>
+Han Boetes <han@boetes.org>
+HansK-p <42314815+HansK-p@users.noreply.github.com>
+Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
 Heiko Zuerker (Smiley73) <heiko@zuerker.org>
+Hugo Locurcio <hugo.locurcio@hugo.pro>
+Iain Barnett <iainspeed@gmail.com>
+Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
+ignacy123 <ignacy.buczek@onet.pl>
+Ikko Ashimine <eltociear@gmail.com>
+Ilya Brin <464157+ilyabrin@users.noreply.github.com>
+Iskander Sharipov (Alex) <quasilyte@gmail.com>
 Jaakko Hannikainen (jgke) <jgke@jgke.fi>
 Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
+Jack Croft <jccroft1@users.noreply.github.com>
+Jacob <jyundt@gmail.com>
 Jake Peterson (acogdev) <jake@acogdev.com>
 Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
 James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
+janost <janost@tuta.io>
+Jaroslav Lichtblau <svetlemodry@users.noreply.github.com>
 Jaroslav Malec (dzarda) <dzardacz@gmail.com>
+jaseg <githubaccount@jaseg.net>
+Jauder Ho <jauderho@users.noreply.github.com>
 Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
+Jeffery To <jeffery.to@gmail.com>
+jelle van der Waa <jelle@vdwaa.nl>
 Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
+Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
+Jesse Lucas <jesse@jesselucas.com>
 Jochen Voss (seehuhn) <voss@seehuhn.de>
+Johan Andersson <j@i19.se>
 Johan Vromans (sciurius) <jvromans@squirrel.nl>
+John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
+Jonas Thelemann <e-mail@jonas-thelemann.de>
+Jonathan <artback@protonmail.com> <jonagn@gmail.com>
+Jonathan Cross <jcross@gmail.com>
+Jonta <359397+Jonta@users.noreply.github.com>
 Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
+jtagcat <git-514635f7@jtag.cat> <git-12dbd862@jtag.cat>
+Jörg Thalheim <Mic92@users.noreply.github.com>
+Jędrzej Kula <kula.jedrek@gmail.com>
+Kalle Laine <pahakalle@protonmail.com>
 Karol Różycki (krozycki) <rozycki.karol@gmail.com>
+Kebin Liu <lkebin@gmail.com>
+Keith Turner <kturner@apache.org>
 Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
 Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
 Kevin Allen (ironmig) <kma1660@gmail.com>
+Kevin Bushiri (keevBush) <keevbush@gmail.com> <36192217+keevBush@users.noreply.github.com>
 Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
+klemens <ka7@github.com>
 Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
 Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
+Lars Lehtonen <lars.lehtonen@gmail.com>
+Laurent Arnoud <laurent@spkdev.net>
 Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
 Leo Arias (elopio) <yo@elopio.net>
 Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
 Lode Hoste (Zillode) <zillode@zillode.be>
 Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
+LSmithx2 <42276854+lsmithx2@users.noreply.github.com>
+Lukas Lihotzki <lukas@lihotzki.de>
+luzpaz <luzpaz@users.noreply.github.com>
 Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
 Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
 Marc Pujol (kilburn) <kilburn@la3.org>
 Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
+marco-m <marco.molteni@laposte.net>
+Marcus Legendre <marcus.legendre@gmail.com>
+Mario Majila <mariustshipichik@gmail.com>
 Mark Pulford (mpx) <mark@kyne.com.au>
+Martchus <martchus@gmx.net>
 Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
+Mateusz Ż <thedead4fun@live.com>
+Matic Potočnik <hairyfotr@gmail.com>
 Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
+Matt Robenolt <matt@ydekproductions.com>
+Matteo Ruina <matteo.ruina@gmail.com>
+Maurizio Tomasi <ziotom78@gmail.com>
+Max <github@germancoding.com>
 Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
+MaximAL <almaximal@ya.ru>
+Maxime Thirouin <m@moox.io>
+mclang <1721600+mclang@users.noreply.github.com>
 Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
 Michael Ploujnikov (plouj) <ploujj@gmail.com>
+Michael Rienstra <mrienstra@gmail.com>
 Michael Tilli (pyfisch) <pyfisch@gmail.com>
+MichaIng <micha@dietpi.com>
+Mike Boone <mike@boonedocks.net>
+MikeLund <MikeLund@users.noreply.github.com>
+MikolajTwarog <43782609+MikolajTwarog@users.noreply.github.com>
+Mingxuan Lin <gdlmx@users.noreply.github.com>
+mv1005 <49659413+mv1005@users.noreply.github.com>
 Nate Morrison (nrm21) <natemorrison@gmail.com>
+Naveen <172697+naveensrinivasan@users.noreply.github.com>
+Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
+Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
+Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
+Nicolas Perraut <n.perraut@gmail.com>
 Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
+Nils Jakobi (thunderstorm99) <jakobi.nils@gmail.com>
+NinoM4ster <ninom4ster@gmail.com>
+Nitroretro <43112364+Nitroretro@users.noreply.github.com>
+NoLooseEnds <jon.koslung@gmail.com>
+Oliver Freyermuth <o.freyermuth@googlemail.com>
+otbutz <tbutz@optitool.de>
+Otiel <Otiel@users.noreply.github.com>
+overkill <22098433+0verk1ll@users.noreply.github.com>
+Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
+Pablo <pbaeyens31+github@gmail.com>
 Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
+Paul Brit <paulbrit44@gmail.com>
 Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
+Paweł Rozlach <vespian@users.noreply.github.com>
+perewa <cavalcante.ten@gmail.com>
+Peter Badida <KeyWeeUsr@users.noreply.github.com>
+Peter Dave Hello <hsu@peterdavehello.org>
 Peter Hoeg (peterhoeg) <peter@speartail.com>
+Peter Marquardt (wwwutz) <wwwutz@gmail.com> <wwwutz@googlemail.com>
+Phani Rithvij <phanirithvij2000@gmail.com>
+Phil Davis <phil.davis@inf.org>
 Philippe Schommers (filoozoom) <philippe@schommers.be>
 Phill Luby (pluby) <phill.luby@newredo.com>
+Pier Paolo Ramon <ramonpierre@gmail.com>
 Piotr Bejda (piobpl) <piotrb10@gmail.com>
+Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
+Quentin Hibon <qh.public@yahoo.com>
+Rahmi Pruitt <rjpruitt16@gmail.com>
+red_led <red-led@users.noreply.github.com>
+Richard Hartmann <RichiH@users.noreply.github.com>
 Robert Carosi (nov1n) <robert@carosi.nl>
+Roberto Santalla <roobre@users.noreply.github.com>
+Robin Schoonover <robin@cornhooves.org>
 Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
 Ross Smith II (rasa) <ross@smithii.com>
+rubenbe <github-com-00ff86@vandamme.email>
+Ruslan Yevdokymov <38809160+ruslanye@users.noreply.github.com>
+Ryan Qian <i@bitbili.net>
 Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
 Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
 Scott Klupfel (kluppy) <kluppy@going2blue.com>
+sec65 <106604020+sec65@users.noreply.github.com>
 Sergey Mishin (ralder) <ralder@yandex.ru>
+Shaarad Dalvi <60266155+shaaraddalvi@users.noreply.github.com>
 Simon Frei (imsodin) <freisim93@gmail.com>
+Simon Mwepu <simonmwepu@gmail.com>
+Sly_tom_cat <slytomcat@mail.ru>
 Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
-Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
+Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
+Steven Eckhoff <steven.eckhoff.opensource@gmail.com>
 Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
+Syncthing Automation <automation@syncthing.net>
+Syncthing Release Automation <release@syncthing.net>
 Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
+Thomas Hipp <thomashipp@gmail.com>
 Tim Abell (timabell) <tim@timwise.co.uk>
 Tim Howes (timhowes) <timhowes@berkeley.edu>
+Tobias Klauser <tobias.klauser@gmail.com>
 Tobias Nygren (tnn2) <tnn@nygren.pp.se>
 Tobias Tom (tobiastom) <t.tom@succont.de>
-Tomas Cerveny (kozec) <kozec@kozec.com>
+Tom Jakubowski <tom@crystae.net>
+Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com> <twilczynski@naver.com>
+Tommy Thorn <tommy-github-email@thorn.ws>
 Tully Robinson (tojrobinson) <tully@tojr.org>
 Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
+Tyler Kropp <kropptyler@gmail.com>
 Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
 Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
 Victor Buinsky (buinsky) <vix_booja@tut.by>
 Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
+villekalliomaki <53118179+villekalliomaki@users.noreply.github.com>
+Vladimir Rusinov <vrusinov@google.com> <vladimir.rusinov@gmail.com>
+wangguoliang <liangcszzu@163.com>
 William A. Kennington III (wkennington) <william@wkennington.com>
+wouter bolsterlee <wouter@bolsterl.ee>
 Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
+xarx00 <xarx00@users.noreply.github.com>
 Xavier O. (damajor) <damajor@gmail.com>
+xjtdy888 (xjtdy888) <xjtdy888@163.com>
 Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
+佛跳墙 <daoquan@qq.com>
+落心 <luoxin.ttt@gmail.com>

CONDUCT.md

@@ -1,92 +1,73 @@
-## Conduct
+# Contributor Covenant Code of Conduct
 
-* We are committed to providing a friendly, safe and welcoming
-  environment for all, regardless of gender, sexual orientation,
-  disability, ethnicity, religion, or similar personal characteristic.
+## Our Pledge
 
-* On IRC, please avoid using overtly sexual nicknames or other nicknames
-  that might detract from a friendly, safe and welcoming environment for
-  all.
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+education, socio-economic status, nationality, personal appearance, race,
+religion, or sexual identity and orientation.
 
-* Please be kind and courteous. There's no need to be mean or rude.
+## Our Standards
 
-* Respect that people have differences of opinion and that every design
-  or implementation choice carries a trade-off and numerous costs. There
-  is seldom a right answer.
+Examples of behavior that contributes to creating a positive environment
+include:
 
-* Please keep unstructured critique to a minimum. If you have solid
-  ideas you want to experiment with, make a fork and see how it works.
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
 
-* We will exclude you from interaction if you insult, demean or harass
-  anyone. That is not welcome behaviour. We interpret the term
-  "harassment" as including the definition in the <a
-  href="http://citizencodeofconduct.org/">Citizen Code of Conduct</a>;
-  if you have any lack of clarity about what might be included in that
-  concept, please read their definition. In particular, we don't
-  tolerate behavior that excludes people in socially marginalized
-  groups.
+Examples of unacceptable behavior by participants include:
 
-* Private harassment is also unacceptable. No matter who you are, if you
-  feel you have been or are being harassed or made uncomfortable by a
-  community member, please contact one of the channel ops or any of the
-  Syncthing core team immediately. Whether you're a regular  contributor
-  or a newcomer, we care about making this community a safe place for
-  you and we've got your back.
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
 
-* Likewise any spamming, trolling, flaming, baiting or other
-  attention-stealing behaviour is not welcome.
+## Our Responsibilities
 
-## Moderation
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
 
-These are the policies for upholding our community's standards of
-conduct in our communication channels, most notably in Syncthing-related
-IRC channels and on the web forum.
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
 
-1. Remarks that violate the Syncthing standards of conduct, including
-   hateful, hurtful, oppressive, or exclusionary remarks, are not
-   allowed. (Cursing is allowed, but never targeting another user, and
-   never in a hateful manner.)
+## Scope
 
-2. Remarks that moderators find inappropriate, whether listed in the
-   code of conduct or not, are also not allowed.
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
 
-3. Moderators will first respond to such remarks with a warning.
+## Enforcement
 
-4. If the warning is unheeded, the user will be "kicked," i.e., kicked
-   out of the communication channel to cool off.
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at security@syncthing.net. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
 
-5. If the user comes back and continues to make trouble, they will be
-   banned, i.e., indefinitely excluded.
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
 
-6. Moderators may choose at their discretion to un-ban the user if it
-   was a first offense and they offer the offended party a genuine
-   apology.
+## Attribution
 
-7. If a moderator bans someone and you think it was unjustified, please
-   take it up with that moderator, or with a different moderator, **in
-   private**. Complaints about bans in-channel are not allowed.
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
 
-8. Moderators are held to a higher standard than other community
-   members. If a moderator creates an inappropriate situation, they
-   should expect less leeway than others.
-
-In the Syncthing community we strive to go the extra step to look out
-for each other. Don't just aim to be technically unimpeachable, try to
-be your best self. In particular, avoid flirting with offensive or
-sensitive issues, particularly if they're off-topic; this all too
-often leads to unnecessary fights, hurt feelings, and damaged trust;
-worse, it can drive people away from the community entirely.
-
-And if someone takes issue with something you said or did, resist the
-urge to be defensive. Just stop doing what it was they complained about
-and apologize. Even if you feel you were misinterpreted or unfairly
-accused, chances are good there was something you could've communicated
-better — remember that it's your responsibility to make your fellow
-community members comfortable. Everyone wants to get along and we are
-all here first and foremost because we want to talk about cool
-technology. You will find that people will be eager to assume good
-intent and forgive as long as you earn their trust.
-
-*Adapted from the [Rust Code of Conduct](https://github.com/rust-lang/rust/wiki/Note-development-policy#conduct)*
-
-*Adapted from the [Node.js Policy on Trolling](http://blog.izs.me/post/30036893703/policy-on-trolling)*
+[homepage]: https://www.contributor-covenant.org

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 ## Reporting Bugs
 
-Please file bugs in the [Github Issue
+Please file bugs in the [GitHub Issue
 Tracker](https://github.com/syncthing/syncthing/issues). Include at
 least the following:
 

Dockerfile

@@ -0,0 +1,29 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f syncthing && go run build.go -no-upgrade build syncthing
+
+FROM alpine
+
+EXPOSE 8384 22000/tcp 22000/udp 21027/udp
+
+VOLUME ["/var/syncthing"]
+
+RUN apk add --no-cache ca-certificates su-exec tzdata libcap
+
+COPY --from=builder /src/syncthing /bin/syncthing
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/syncthing
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z 127.0.0.1 8384 || exit 1
+
+ENV STGUIADDRESS=0.0.0.0:8384
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]

Dockerfile.builder

@@ -0,0 +1,9 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION
+
+# FPM to build Debian packages
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	locales rubygems ruby-dev build-essential git \
+	&& apt-get clean \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& gem install fpm

Dockerfile.buildx

@@ -0,0 +1,19 @@
+FROM alpine
+ARG TARGETARCH
+
+EXPOSE 8384 22000/tcp 22000/udp 21027/udp
+
+VOLUME ["/var/syncthing"]
+
+RUN apk add --no-cache ca-certificates su-exec tzdata
+
+COPY ./syncthing-linux-$TARGETARCH /bin/syncthing
+COPY ./script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/syncthing
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z 127.0.0.1 8384 || exit 1
+
+ENV STGUIADDRESS=0.0.0.0:8384
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]

Dockerfile.stdiscosrv

@@ -0,0 +1,29 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f stdiscosrv && go run build.go -no-upgrade build stdiscosrv
+
+FROM alpine
+
+EXPOSE 19200 8443
+
+VOLUME ["/var/stdiscosrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/stdiscosrv /bin/stdiscosrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/stdiscosrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 8443 || exit 1
+
+WORKDIR /var/stdiscosrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/stdiscosrv"]

Dockerfile.strelaysrv

@@ -0,0 +1,29 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f strelaysrv && go run build.go -no-upgrade build strelaysrv
+
+FROM alpine
+
+EXPOSE 22067 22070
+
+VOLUME ["/var/strelaysrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/strelaysrv /bin/strelaysrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/strelaysrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 22067 || exit 1
+
+WORKDIR /var/strelaysrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaysrv"]

GOALS.md

@@ -57,7 +57,7 @@ latest technology is not always available to any given individual.
 > Computers include desktops, laptops, servers, virtual machines, small
 > general purpose computers such as Raspberry Pis and, *where possible*,
 > tablets and phones. NAS appliances, toasters, cars, firearms, thermostats
-> and so on may include computing capabitilies but it is not our goal for
+> and so on may include computing capabilities but it is not our goal for
 > Syncthing to run smoothly on these devices.
 
 ### 6. For Individuals

ISSUE_TEMPLATE.md

@@ -1,34 +0,0 @@
-### DO NOT REPORT SECURITY ISSUES IN THIS ISSUE TRACKER
-
-Instead, contact security@syncthing.net directly - see
-https://syncthing.net/security.html for more information.
-
-### DO NOT POST SUPPORT REQUESTS OR GENERAL QUESTIONS IN THIS ISSUE TRACKER
-
-Please use the forum at https://forum.syncthing.net/ where a large number of
-helpful people hang out. This issue tracker is for reporting bugs or feature
-requests directly to the developers. Worst case you might get a short
-"that's a bug, please report it on GitHub" response on the forum, in which
-case we thank you for your patience and following our advice. :)
-
-### Please do post actual bug reports and feature requests.
-
-If your issue is a bug report, replace this boilerplate with a description
-of the problem, being sure to include at least:
-
- - what happened,
- - what you expected to happen instead, and
- - any steps to reproduce the problem.
-
-Also fill out the version information below and add log output or
-screenshots as appropriate.
-
-If your issue is a feature request, simply replace this template text in
-its entirety.
-
-### Version Information
-
-Syncthing Version: v0.x.y
-OS Version: Windows 7 / Ubuntu 14.04 / ...
-Browser Version: (if applicable, for GUI issues)
-

README-Docker.md

@@ -0,0 +1,109 @@
+# Docker Container for Syncthing
+
+Use the Dockerfile in this repo, or pull the `syncthing/syncthing` image
+from Docker Hub.
+
+Use the `/var/syncthing` volume to have the synchronized files available on the
+host. You can add more folders and map them as you prefer.
+
+Note that Syncthing runs as UID 1000 and GID 1000 by default. These may be
+altered with the `PUID` and `PGID` environment variables. In addition
+the name of the Syncthing instance can be optionally defined by using
+`--hostname=syncthing` parameter.
+
+To grant Syncthing additional capabilities without running as root, use the
+`PCAP` environment variable with the same syntax as that for `setcap(8)`.
+For example, `PCAP=cap_chown,cap_fowner+ep`.
+
+## Example Usage
+
+**Docker cli**
+```
+$ docker pull syncthing/syncthing
+$ docker run -p 8384:8384 -p 22000:22000/tcp -p 22000:22000/udp -p 21027:21027/udp \
+    -v /wherever/st-sync:/var/syncthing \
+    --hostname=my-syncthing \
+    syncthing/syncthing:latest
+```
+
+**Docker compose**
+```yml
+---
+version: "3"
+services:
+  syncthing:
+    image: syncthing/syncthing
+    container_name: syncthing
+    hostname: my-syncthing
+    environment:
+      - PUID=1000
+      - PGID=1000
+    volumes:
+      - /wherever/st-sync:/var/syncthing
+    ports:
+      - 8384:8384 # Web UI
+      - 22000:22000/tcp # TCP file transfers
+      - 22000:22000/udp # QUIC file transfers
+      - 21027:21027/udp # Receive local discovery broadcasts
+    restart: unless-stopped
+```
+
+## Discovery
+
+Note that Docker's default network mode prevents local IP addresses from
+being discovered, as Syncthing is only able to see the internal IP of the
+container on the `172.17.0.0/16` subnet. This will result in poor transfer rates
+if local device addresses are not manually configured.
+
+It is therefore advisable to use the [host network mode](https://docs.docker.com/network/host/) instead:
+
+**Docker cli**
+```
+$ docker pull syncthing/syncthing
+$ docker run --network=host \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+**Docker compose**
+```yml
+---
+version: "3"
+services:
+  syncthing:
+    image: syncthing/syncthing
+    container_name: syncthing
+    hostname: my-syncthing
+    environment:
+      - PUID=1000
+      - PGID=1000
+    volumes:
+      - /wherever/st-sync:/var/syncthing
+    network_mode: host
+    restart: unless-stopped
+```
+
+Be aware that syncthing alone is now in control of what interfaces and ports it
+listens on. You can edit the syncthing configuration to change the defaults if
+there are conflicts.
+
+## GUI Security
+
+By default Syncthing inside the Docker image listens on 0.0.0.0:8384 to
+allow GUI connections via the Docker proxy. This is set by the
+`STGUIADDRESS` environment variable in the Dockerfile, as it differs from
+what Syncthing would otherwise use by default. This means you should set up
+authentication in the GUI, like for any other externally reachable Syncthing
+instance. If you do not require the GUI, or you use host networking, you can
+unset the `STGUIADDRESS` variable to have Syncthing fall back to listening
+on 127.0.0.1:
+
+```
+$ docker pull syncthing/syncthing
+$ docker run -e STGUIADDRESS= \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+With the environment variable unset Syncthing will follow what is set in the
+configuration file / GUI settings dialog.

README.md

@@ -2,7 +2,6 @@
 
 ---
 
-[![Latest Downloads](https://img.shields.io/badge/latest-downloads-brightgreen.svg?style=flat-square)](https://build.syncthing.net/latest/)
 [![Latest Linux & Cross Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildLinuxCross.svg?style=flat-square&label=linux+%26+cross+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildLinuxCross&guest=1)
 [![Latest Windows Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildWindows.svg?style=flat-square&label=windows+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildWindows&guest=1)
 [![Latest Mac Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildMac.svg?style=flat-square&label=mac+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildMac&guest=1)
@@ -63,6 +62,10 @@ There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory][3]. There are also several [GUI
 implementations][11] for Windows, Mac and Linux.
 
+## Docker
+
+To run Syncthing in Docker, see [the Docker README][16].
+
 ## Vote on features/bugs
 
 We'd like to encourage you to [vote][12] on issues that matter to you.
@@ -70,15 +73,16 @@ This helps the team understand what are the biggest pain points for our users, a
 
 ## Getting in Touch
 
-The first and best point of contact is the [Forum][8]. There is also an IRC
-channel, `#syncthing` on [freenode][4] (with a [web client][9]), for talking
-directly to developers and users. If you've found something that is clearly a
+The first and best point of contact is the [Forum][8].
+If you've found something that is clearly a
 bug, feel free to report it in the [GitHub issue tracker][10].
 
 ## Building
 
-Building Syncthing from source is easy, and there's a [guide][5]
-that describes it for both Unix and Windows systems.
+Building Syncthing from source is easy. After extracting the source bundle from
+a release or checking out git, you just need to run `go run build.go` and the
+binaries are created in `./bin`. There's [a guide][5] with more details on the
+build process.
 
 ## Signed Releases
 
@@ -87,28 +91,27 @@ D26E6ED000654A3E, available from https://syncthing.net/security.html and
 most key servers.
 
 There is also a built in automatic upgrade mechanism (disabled in some
-distribution channels) which uses a compiled in ECDSA signature. Mac OS
-X binaries are also properly code signed.
+distribution channels) which uses a compiled in ECDSA signature. macOS
+binaries are also properly code signed.
 
 ## Documentation
 
-Please see the [Syncthing documentation site][6].
+Please see the Syncthing [documentation site][6] [[source]][17].
 
 All code is licensed under the [MPLv2 License][7].
 
 [1]: https://docs.syncthing.net/specs/bep-v1.html
 [2]: https://docs.syncthing.net/intro/getting-started.html
-[3]: https://github.com/syncthing/syncthing/blob/master/etc
-[4]: https://www.freenode.net/
+[3]: https://github.com/syncthing/syncthing/blob/main/etc
 [5]: https://docs.syncthing.net/dev/building.html
 [6]: https://docs.syncthing.net/
-[7]: https://github.com/syncthing/syncthing/blob/master/LICENSE
+[7]: https://github.com/syncthing/syncthing/blob/main/LICENSE
 [8]: https://forum.syncthing.net/
-[9]: https://kiwiirc.com/client/irc.freenode.net/#syncthing
 [10]: https://github.com/syncthing/syncthing/issues
 [11]: https://docs.syncthing.net/users/contrib.html#gui-wrappers
 [12]: https://www.bountysource.com/teams/syncthing/issues
-[13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
+[13]: https://github.com/syncthing/syncthing/blob/main/GOALS.md
 [14]: assets/logo-text-128.png
 [15]: https://syncthing.net/
-
+[16]: https://github.com/syncthing/syncthing/blob/main/README-Docker.md
+[17]: https://github.com/syncthing/docs

build.ps1

@@ -0,0 +1,20 @@
+function build {
+    go run build.go @args
+}
+
+$cmd, $rest = $args
+switch ($cmd) {
+    "test" {
+        $env:LOGGER_DISCARD=1
+        build test
+    }
+
+    "bench" {
+        $env:LOGGER_DISCARD=1
+        build bench
+    }
+
+    default {
+        build @rest
+    }
+}

build.sh

@@ -2,8 +2,6 @@
 set -euo pipefail
 IFS=$'\n\t'
 
-STTRACE=${STTRACE:-}
-
 script() {
 	name="$1"
 	shift
@@ -15,94 +13,23 @@ build() {
 }
 
 case "${1:-default}" in
-	default)
-		build
-		;;
-
-	clean)
-		build "$@"
-		;;
-
-	tar)
-		build "$@"
-		;;
-
-	assets)
-		build "$@"
-		;;
-
-	xdr)
-		build "$@"
-		;;
-
-	translate)
-		build "$@"
-		;;
-
-	deb)
-		build "$@"
-		;;
-
-	setup)
-		build "$@"
-		;;
-
 	test)
-		ulimit -t 600 &>/dev/null || true
-		ulimit -d 512000 &>/dev/null || true
-		ulimit -m 512000 &>/dev/null || true
 		LOGGER_DISCARD=1 build test
 		;;
 
 	bench)
-		LOGGER_DISCARD=1 build bench | script benchfilter
+		LOGGER_DISCARD=1 build bench
 		;;
 
 	prerelease)
-		go run script/authors.go
+		script authors
 		build transifex
 		pushd man ; ./refresh.sh ; popd
-		git add -A gui man
-		git commit -m 'gui, man: Update docs & translations'
-		;;
-
-	noupgrade)
-		build -no-upgrade tar
-		;;
-
-	all)
-		platforms=(
-			darwin-amd64 dragonfly-amd64 freebsd-amd64 linux-amd64 netbsd-amd64 openbsd-amd64 solaris-amd64 windows-amd64
-			freebsd-386 linux-386 netbsd-386 openbsd-386 windows-386
-			linux-arm linux-arm64 linux-ppc64 linux-ppc64le
-		)
-
-		for plat in "${platforms[@]}"; do
-			echo Building "$plat"
-
-			goos="${plat%-*}"
-			goarch="${plat#*-}"
-			dist="tar"
-
-			if [[ $goos == "windows" ]]; then
-				dist="zip"
-			fi
-
-			build -goos "$goos" -goarch "$goarch" "$dist"
-			echo
-		done
-		;;
-
-	test-xunit)
-		ulimit -t 600 &>/dev/null || true
-		ulimit -d 512000 &>/dev/null || true
-		ulimit -m 512000 &>/dev/null || true
-
-		(GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -v -race ./lib/... ./cmd/... || true) > tests.out
-		go2xunit -output tests.xml -fail < tests.out
+		git add -A gui man AUTHORS
+		git commit -m 'gui, man, authors: Update docs, translations, and contributors'
 		;;
 
 	*)
-		echo "Unknown build command $1"
+		build "$@"
 		;;
 esac

cmd/stbench/main.go

@@ -1,143 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// This doesn't build on Windows due to the Rusage stuff.
-
-// +build !windows
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"runtime"
-	"syscall"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/rc"
-)
-
-var homeDir = "h1"
-var syncthingBin = "./bin/syncthing"
-var test = "scan"
-
-func main() {
-	flag.StringVar(&homeDir, "home", homeDir, "Home directory location")
-	flag.StringVar(&syncthingBin, "bin", syncthingBin, "Binary location")
-	flag.StringVar(&test, "test", test, "Test to run")
-	flag.Parse()
-
-	switch test {
-	case "scan":
-		// scan measures the resource usage required to perform the initial
-		// scan, without cleaning away the database first.
-		testScan()
-	}
-}
-
-// testScan starts a process and reports on the resource usage required to
-// perform the initial scan.
-func testScan() {
-	log.Println("Starting...")
-	p := rc.NewProcess("127.0.0.1:8081")
-	if err := p.Start(syncthingBin, "-home", homeDir, "-no-browser"); err != nil {
-		log.Println(err)
-		return
-	}
-	defer p.Stop()
-
-	wallTime := awaitScanComplete(p)
-
-	report(p, wallTime)
-}
-
-// awaitScanComplete waits for a folder to transition idle->scanning and
-// then scanning->idle and returns the time taken for the scan.
-func awaitScanComplete(p *rc.Process) time.Duration {
-	log.Println("Awaiting scan completion...")
-	var t0, t1 time.Time
-	lastEvent := 0
-loop:
-	for {
-		evs, err := p.Events(lastEvent)
-		if err != nil {
-			continue
-		}
-
-		for _, ev := range evs {
-			if ev.Type == "StateChanged" {
-				data := ev.Data.(map[string]interface{})
-				log.Println(ev)
-
-				if data["to"].(string) == "scanning" {
-					t0 = ev.Time
-					continue
-				}
-
-				if !t0.IsZero() && data["to"].(string) == "idle" {
-					t1 = ev.Time
-					break loop
-				}
-			}
-			lastEvent = ev.ID
-		}
-
-		time.Sleep(250 * time.Millisecond)
-	}
-
-	return t1.Sub(t0)
-}
-
-// report stops the given process and reports on its resource usage in two
-// ways: human readable to stderr, and CSV to stdout.
-func report(p *rc.Process, wallTime time.Duration) {
-	sv, err := p.SystemVersion()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	ss, err := p.SystemStatus()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	proc, err := p.Stop()
-	if err != nil {
-		return
-	}
-
-	rusage, ok := proc.SysUsage().(*syscall.Rusage)
-	if !ok {
-		return
-	}
-
-	log.Println("Version:", sv.Version)
-	log.Println("Alloc:", ss.Alloc/1024, "KiB")
-	log.Println("Sys:", ss.Sys/1024, "KiB")
-	log.Println("Goroutines:", ss.Goroutines)
-	log.Println("Wall time:", wallTime)
-	log.Println("Utime:", time.Duration(rusage.Utime.Nano()))
-	log.Println("Stime:", time.Duration(rusage.Stime.Nano()))
-	if runtime.GOOS == "darwin" {
-		// Darwin reports in bytes, Linux seems to report in KiB even
-		// though the manpage says otherwise.
-		rusage.Maxrss /= 1024
-	}
-	log.Println("MaxRSS:", rusage.Maxrss, "KiB")
-
-	fmt.Printf("%s,%d,%d,%d,%.02f,%.02f,%.02f,%d\n",
-		sv.Version,
-		ss.Alloc/1024,
-		ss.Sys/1024,
-		ss.Goroutines,
-		wallTime.Seconds(),
-		time.Duration(rusage.Utime.Nano()).Seconds(),
-		time.Duration(rusage.Stime.Nano()).Seconds(),
-		rusage.Maxrss)
-}

cmd/stcli/LICENSE

@@ -1,19 +0,0 @@
-Copyright (C) 2014 Audrius Butkevičius
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-- The above copyright notice and this permission notice shall be included in
-  all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

cmd/stcli/client.go

@@ -1,115 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"bytes"
-	"crypto/tls"
-	"net/http"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-type APIClient struct {
-	httpClient http.Client
-	endpoint   string
-	apikey     string
-	username   string
-	password   string
-	id         string
-	csrf       string
-}
-
-var instance *APIClient
-
-func getClient(c *cli.Context) *APIClient {
-	if instance != nil {
-		return instance
-	}
-	endpoint := c.GlobalString("endpoint")
-	if !strings.HasPrefix(endpoint, "http") {
-		endpoint = "http://" + endpoint
-	}
-	httpClient := http.Client{
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: c.GlobalBool("insecure"),
-			},
-		},
-	}
-	client := APIClient{
-		httpClient: httpClient,
-		endpoint:   endpoint,
-		apikey:     c.GlobalString("apikey"),
-		username:   c.GlobalString("username"),
-		password:   c.GlobalString("password"),
-	}
-
-	if client.apikey == "" {
-		request, err := http.NewRequest("GET", client.endpoint, nil)
-		die(err)
-		response := client.handleRequest(request)
-		client.id = response.Header.Get("X-Syncthing-ID")
-		if client.id == "" {
-			die("Failed to get device ID")
-		}
-		for _, item := range response.Cookies() {
-			if item.Name == "CSRF-Token-"+client.id[:5] {
-				client.csrf = item.Value
-				goto csrffound
-			}
-		}
-		die("Failed to get CSRF token")
-	csrffound:
-	}
-	instance = &client
-	return &client
-}
-
-func (client *APIClient) handleRequest(request *http.Request) *http.Response {
-	if client.apikey != "" {
-		request.Header.Set("X-API-Key", client.apikey)
-	}
-	if client.username != "" || client.password != "" {
-		request.SetBasicAuth(client.username, client.password)
-	}
-	if client.csrf != "" {
-		request.Header.Set("X-CSRF-Token-"+client.id[:5], client.csrf)
-	}
-
-	response, err := client.httpClient.Do(request)
-	die(err)
-
-	if response.StatusCode == 404 {
-		die("Invalid endpoint or API call")
-	} else if response.StatusCode == 401 {
-		die("Invalid username or password")
-	} else if response.StatusCode == 403 {
-		if client.apikey == "" {
-			die("Invalid CSRF token")
-		}
-		die("Invalid API key")
-	} else if response.StatusCode != 200 {
-		body := strings.TrimSpace(string(responseToBArray(response)))
-		if body != "" {
-			die(body)
-		}
-		die("Unknown HTTP status returned: " + response.Status)
-	}
-	return response
-}
-
-func httpGet(c *cli.Context, url string) *http.Response {
-	client := getClient(c)
-	request, err := http.NewRequest("GET", client.endpoint+"/rest/"+url, nil)
-	die(err)
-	return client.handleRequest(request)
-}
-
-func httpPost(c *cli.Context, url string, body string) *http.Response {
-	client := getClient(c)
-	request, err := http.NewRequest("POST", client.endpoint+"/rest/"+url, bytes.NewBufferString(body))
-	die(err)
-	return client.handleRequest(request)
-}

cmd/stcli/cmd_devices.go

@@ -1,188 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-	"github.com/syncthing/syncthing/lib/config"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "devices",
-		HideHelp: true,
-		Usage:    "Device command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "list",
-				Usage:    "List registered devices",
-				Requires: &cli.Requires{},
-				Action:   devicesList,
-			},
-			{
-				Name:     "add",
-				Usage:    "Add a new device",
-				Requires: &cli.Requires{"device id", "device name?"},
-				Action:   devicesAdd,
-			},
-			{
-				Name:     "remove",
-				Usage:    "Remove an existing device",
-				Requires: &cli.Requires{"device id"},
-				Action:   devicesRemove,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a property of a device",
-				Requires: &cli.Requires{"device id", "property"},
-				Action:   devicesGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a property of a device",
-				Requires: &cli.Requires{"device id", "property", "value..."},
-				Action:   devicesSet,
-			},
-		},
-	})
-}
-
-func devicesList(c *cli.Context) {
-	cfg := getConfig(c)
-	first := true
-	writer := newTableWriter()
-	for _, device := range cfg.Devices {
-		if !first {
-			fmt.Fprintln(writer)
-		}
-		fmt.Fprintln(writer, "ID:\t", device.DeviceID, "\t")
-		fmt.Fprintln(writer, "Name:\t", device.Name, "\t(name)")
-		fmt.Fprintln(writer, "Address:\t", strings.Join(device.Addresses, " "), "\t(address)")
-		fmt.Fprintln(writer, "Compression:\t", device.Compression, "\t(compression)")
-		fmt.Fprintln(writer, "Certificate name:\t", device.CertName, "\t(certname)")
-		fmt.Fprintln(writer, "Introducer:\t", device.Introducer, "\t(introducer)")
-		first = false
-	}
-	writer.Flush()
-}
-
-func devicesAdd(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-
-	newDevice := config.DeviceConfiguration{
-		DeviceID:  id,
-		Name:      nid,
-		Addresses: []string{"dynamic"},
-	}
-
-	if len(c.Args()) > 1 {
-		newDevice.Name = c.Args()[1]
-	}
-
-	if len(c.Args()) > 2 {
-		addresses := c.Args()[2:]
-		for _, item := range addresses {
-			if item == "dynamic" {
-				continue
-			}
-			validAddress(item)
-		}
-		newDevice.Addresses = addresses
-	}
-
-	cfg := getConfig(c)
-	for _, device := range cfg.Devices {
-		if device.DeviceID == id {
-			die("Device " + nid + " already exists")
-		}
-	}
-	cfg.Devices = append(cfg.Devices, newDevice)
-	setConfig(c, cfg)
-}
-
-func devicesRemove(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-	if nid == getMyID(c) {
-		die("Cannot remove yourself")
-	}
-	cfg := getConfig(c)
-	for i, device := range cfg.Devices {
-		if device.DeviceID == id {
-			last := len(cfg.Devices) - 1
-			cfg.Devices[i] = cfg.Devices[last]
-			cfg.Devices = cfg.Devices[:last]
-			setConfig(c, cfg)
-			return
-		}
-	}
-	die("Device " + nid + " not found")
-}
-
-func devicesGet(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-	arg := c.Args()[1]
-	cfg := getConfig(c)
-	for _, device := range cfg.Devices {
-		if device.DeviceID != id {
-			continue
-		}
-		switch strings.ToLower(arg) {
-		case "name":
-			fmt.Println(device.Name)
-		case "address":
-			fmt.Println(strings.Join(device.Addresses, "\n"))
-		case "compression":
-			fmt.Println(device.Compression.String())
-		case "certname":
-			fmt.Println(device.CertName)
-		case "introducer":
-			fmt.Println(device.Introducer)
-		default:
-			die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
-		}
-		return
-	}
-	die("Device " + nid + " not found")
-}
-
-func devicesSet(c *cli.Context) {
-	nid := c.Args()[0]
-	id := parseDeviceID(nid)
-	arg := c.Args()[1]
-	config := getConfig(c)
-	for i, device := range config.Devices {
-		if device.DeviceID != id {
-			continue
-		}
-		switch strings.ToLower(arg) {
-		case "name":
-			config.Devices[i].Name = strings.Join(c.Args()[2:], " ")
-		case "address":
-			for _, item := range c.Args()[2:] {
-				if item == "dynamic" {
-					continue
-				}
-				validAddress(item)
-			}
-			config.Devices[i].Addresses = c.Args()[2:]
-		case "compression":
-			err := config.Devices[i].Compression.UnmarshalText([]byte(c.Args()[2]))
-			die(err)
-		case "certname":
-			config.Devices[i].CertName = strings.Join(c.Args()[2:], " ")
-		case "introducer":
-			config.Devices[i].Introducer = parseBool(c.Args()[2])
-		default:
-			die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
-		}
-		setConfig(c, config)
-		return
-	}
-	die("Device " + nid + " not found")
-}

cmd/stcli/cmd_errors.go

@@ -1,67 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "errors",
-		HideHelp: true,
-		Usage:    "Error command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "show",
-				Usage:    "Show pending errors",
-				Requires: &cli.Requires{},
-				Action:   errorsShow,
-			},
-			{
-				Name:     "push",
-				Usage:    "Push an error to active clients",
-				Requires: &cli.Requires{"error message..."},
-				Action:   errorsPush,
-			},
-			{
-				Name:     "clear",
-				Usage:    "Clear pending errors",
-				Requires: &cli.Requires{},
-				Action:   wrappedHTTPPost("system/error/clear"),
-			},
-		},
-	})
-}
-
-func errorsShow(c *cli.Context) {
-	response := httpGet(c, "system/error")
-	var data map[string][]map[string]interface{}
-	json.Unmarshal(responseToBArray(response), &data)
-	writer := newTableWriter()
-	for _, item := range data["errors"] {
-		time := item["time"].(string)[:19]
-		time = strings.Replace(time, "T", " ", 1)
-		err := item["error"].(string)
-		err = strings.TrimSpace(err)
-		fmt.Fprintln(writer, time+":\t"+err)
-	}
-	writer.Flush()
-}
-
-func errorsPush(c *cli.Context) {
-	err := strings.Join(c.Args(), " ")
-	response := httpPost(c, "system/error", strings.TrimSpace(err))
-	if response.StatusCode != 200 {
-		err = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
-		body := string(responseToBArray(response))
-		if body != "" {
-			err += "\nBody: " + body
-		}
-		die(err)
-	}
-}

cmd/stcli/cmd_folders.go

@@ -1,361 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"path/filepath"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/fs"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "folders",
-		HideHelp: true,
-		Usage:    "Folder command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "list",
-				Usage:    "List available folders",
-				Requires: &cli.Requires{},
-				Action:   foldersList,
-			},
-			{
-				Name:     "add",
-				Usage:    "Add a new folder",
-				Requires: &cli.Requires{"folder id", "directory"},
-				Action:   foldersAdd,
-			},
-			{
-				Name:     "remove",
-				Usage:    "Remove an existing folder",
-				Requires: &cli.Requires{"folder id"},
-				Action:   foldersRemove,
-			},
-			{
-				Name:     "override",
-				Usage:    "Override changes from other nodes for a master folder",
-				Requires: &cli.Requires{"folder id"},
-				Action:   foldersOverride,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a property of a folder",
-				Requires: &cli.Requires{"folder id", "property"},
-				Action:   foldersGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a property of a folder",
-				Requires: &cli.Requires{"folder id", "property", "value..."},
-				Action:   foldersSet,
-			},
-			{
-				Name:     "unset",
-				Usage:    "Unset a property of a folder",
-				Requires: &cli.Requires{"folder id", "property"},
-				Action:   foldersUnset,
-			},
-			{
-				Name:     "devices",
-				Usage:    "Folder devices command group",
-				HideHelp: true,
-				Subcommands: []cli.Command{
-					{
-						Name:     "list",
-						Usage:    "List of devices which the folder is shared with",
-						Requires: &cli.Requires{"folder id"},
-						Action:   foldersDevicesList,
-					},
-					{
-						Name:     "add",
-						Usage:    "Share a folder with a device",
-						Requires: &cli.Requires{"folder id", "device id"},
-						Action:   foldersDevicesAdd,
-					},
-					{
-						Name:     "remove",
-						Usage:    "Unshare a folder with a device",
-						Requires: &cli.Requires{"folder id", "device id"},
-						Action:   foldersDevicesRemove,
-					},
-					{
-						Name:     "clear",
-						Usage:    "Unshare a folder with all devices",
-						Requires: &cli.Requires{"folder id"},
-						Action:   foldersDevicesClear,
-					},
-				},
-			},
-		},
-	})
-}
-
-func foldersList(c *cli.Context) {
-	cfg := getConfig(c)
-	first := true
-	writer := newTableWriter()
-	for _, folder := range cfg.Folders {
-		if !first {
-			fmt.Fprintln(writer)
-		}
-		fs := folder.Filesystem()
-		fmt.Fprintln(writer, "ID:\t", folder.ID, "\t")
-		fmt.Fprintln(writer, "Path:\t", fs.URI(), "\t(directory)")
-		fmt.Fprintln(writer, "Path type:\t", fs.Type(), "\t(directory-type)")
-		fmt.Fprintln(writer, "Folder type:\t", folder.Type, "\t(type)")
-		fmt.Fprintln(writer, "Ignore permissions:\t", folder.IgnorePerms, "\t(permissions)")
-		fmt.Fprintln(writer, "Rescan interval in seconds:\t", folder.RescanIntervalS, "\t(rescan)")
-
-		if folder.Versioning.Type != "" {
-			fmt.Fprintln(writer, "Versioning:\t", folder.Versioning.Type, "\t(versioning)")
-			for key, value := range folder.Versioning.Params {
-				fmt.Fprintf(writer, "Versioning %s:\t %s \t(versioning-%s)\n", key, value, key)
-			}
-		}
-		first = false
-	}
-	writer.Flush()
-}
-
-func foldersAdd(c *cli.Context) {
-	cfg := getConfig(c)
-	abs, err := filepath.Abs(c.Args()[1])
-	die(err)
-	folder := config.FolderConfiguration{
-		ID:             c.Args()[0],
-		Path:           filepath.Clean(abs),
-		FilesystemType: fs.FilesystemTypeBasic,
-	}
-	cfg.Folders = append(cfg.Folders, folder)
-	setConfig(c, cfg)
-}
-
-func foldersRemove(c *cli.Context) {
-	cfg := getConfig(c)
-	rid := c.Args()[0]
-	for i, folder := range cfg.Folders {
-		if folder.ID == rid {
-			last := len(cfg.Folders) - 1
-			cfg.Folders[i] = cfg.Folders[last]
-			cfg.Folders = cfg.Folders[:last]
-			setConfig(c, cfg)
-			return
-		}
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersOverride(c *cli.Context) {
-	cfg := getConfig(c)
-	rid := c.Args()[0]
-	for _, folder := range cfg.Folders {
-		if folder.ID == rid && folder.Type == config.FolderTypeSendOnly {
-			response := httpPost(c, "db/override", "")
-			if response.StatusCode != 200 {
-				err := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
-				body := string(responseToBArray(response))
-				if body != "" {
-					err += "\nBody: " + body
-				}
-				die(err)
-			}
-			return
-		}
-	}
-	die("Folder " + rid + " not found or folder not master")
-}
-
-func foldersGet(c *cli.Context) {
-	cfg := getConfig(c)
-	rid := c.Args()[0]
-	arg := strings.ToLower(c.Args()[1])
-	for _, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		if strings.HasPrefix(arg, "versioning-") {
-			arg = arg[11:]
-			value, ok := folder.Versioning.Params[arg]
-			if ok {
-				fmt.Println(value)
-				return
-			}
-			die("Versioning property " + c.Args()[1][11:] + " not found")
-		}
-		switch arg {
-		case "directory":
-			fmt.Println(folder.Filesystem().URI())
-		case "directory-type":
-			fmt.Println(folder.Filesystem().Type())
-		case "type":
-			fmt.Println(folder.Type)
-		case "permissions":
-			fmt.Println(folder.IgnorePerms)
-		case "rescan":
-			fmt.Println(folder.RescanIntervalS)
-		case "versioning":
-			if folder.Versioning.Type != "" {
-				fmt.Println(folder.Versioning.Type)
-			}
-		default:
-			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, directory-type, type, permissions, versioning, versioning-<key>")
-		}
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersSet(c *cli.Context) {
-	rid := c.Args()[0]
-	arg := strings.ToLower(c.Args()[1])
-	val := strings.Join(c.Args()[2:], " ")
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		if strings.HasPrefix(arg, "versioning-") {
-			cfg.Folders[i].Versioning.Params[arg[11:]] = val
-			setConfig(c, cfg)
-			return
-		}
-		switch arg {
-		case "directory":
-			cfg.Folders[i].Path = val
-		case "directory-type":
-			var fsType fs.FilesystemType
-			fsType.UnmarshalText([]byte(val))
-			cfg.Folders[i].FilesystemType = fsType
-		case "type":
-			var t config.FolderType
-			if err := t.UnmarshalText([]byte(val)); err != nil {
-				die("Invalid folder type: " + err.Error())
-			}
-			cfg.Folders[i].Type = t
-		case "permissions":
-			cfg.Folders[i].IgnorePerms = parseBool(val)
-		case "rescan":
-			cfg.Folders[i].RescanIntervalS = parseInt(val)
-		case "versioning":
-			cfg.Folders[i].Versioning.Type = val
-		default:
-			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, master, permissions, versioning, versioning-<key>")
-		}
-		setConfig(c, cfg)
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersUnset(c *cli.Context) {
-	rid := c.Args()[0]
-	arg := strings.ToLower(c.Args()[1])
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		if strings.HasPrefix(arg, "versioning-") {
-			arg = arg[11:]
-			if _, ok := folder.Versioning.Params[arg]; ok {
-				delete(cfg.Folders[i].Versioning.Params, arg)
-				setConfig(c, cfg)
-				return
-			}
-			die("Versioning property " + c.Args()[1][11:] + " not found")
-		}
-		switch arg {
-		case "versioning":
-			cfg.Folders[i].Versioning.Type = ""
-			cfg.Folders[i].Versioning.Params = make(map[string]string)
-		default:
-			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: versioning, versioning-<key>")
-		}
-		setConfig(c, cfg)
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesList(c *cli.Context) {
-	rid := c.Args()[0]
-	cfg := getConfig(c)
-	for _, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		for _, device := range folder.Devices {
-			fmt.Println(device.DeviceID)
-		}
-		return
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesAdd(c *cli.Context) {
-	rid := c.Args()[0]
-	nid := parseDeviceID(c.Args()[1])
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		for _, device := range folder.Devices {
-			if device.DeviceID == nid {
-				die("Device " + c.Args()[1] + " is already part of this folder")
-			}
-		}
-		for _, device := range cfg.Devices {
-			if device.DeviceID == nid {
-				cfg.Folders[i].Devices = append(folder.Devices, config.FolderDeviceConfiguration{
-					DeviceID: device.DeviceID,
-				})
-				setConfig(c, cfg)
-				return
-			}
-		}
-		die("Device " + c.Args()[1] + " not found in device list")
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesRemove(c *cli.Context) {
-	rid := c.Args()[0]
-	nid := parseDeviceID(c.Args()[1])
-	cfg := getConfig(c)
-	for ri, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		for ni, device := range folder.Devices {
-			if device.DeviceID == nid {
-				last := len(folder.Devices) - 1
-				cfg.Folders[ri].Devices[ni] = folder.Devices[last]
-				cfg.Folders[ri].Devices = cfg.Folders[ri].Devices[:last]
-				setConfig(c, cfg)
-				return
-			}
-		}
-		die("Device " + c.Args()[1] + " not found")
-	}
-	die("Folder " + rid + " not found")
-}
-
-func foldersDevicesClear(c *cli.Context) {
-	rid := c.Args()[0]
-	cfg := getConfig(c)
-	for i, folder := range cfg.Folders {
-		if folder.ID != rid {
-			continue
-		}
-		cfg.Folders[i].Devices = []config.FolderDeviceConfiguration{}
-		setConfig(c, cfg)
-		return
-	}
-	die("Folder " + rid + " not found")
-}

cmd/stcli/cmd_general.go

@@ -1,78 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, []cli.Command{
-		{
-			Name:     "id",
-			Usage:    "Get ID of the Syncthing client",
-			Requires: &cli.Requires{},
-			Action:   generalID,
-		},
-		{
-			Name:     "status",
-			Usage:    "Configuration status, whether or not a restart is required for changes to take effect",
-			Requires: &cli.Requires{},
-			Action:   generalStatus,
-		},
-		{
-			Name:     "restart",
-			Usage:    "Restart syncthing",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/restart"),
-		},
-		{
-			Name:     "shutdown",
-			Usage:    "Shutdown syncthing",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/shutdown"),
-		},
-		{
-			Name:     "reset",
-			Usage:    "Reset syncthing deleting all folders and devices",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/reset"),
-		},
-		{
-			Name:     "upgrade",
-			Usage:    "Upgrade syncthing (if a newer version is available)",
-			Requires: &cli.Requires{},
-			Action:   wrappedHTTPPost("system/upgrade"),
-		},
-		{
-			Name:     "version",
-			Usage:    "Syncthing client version",
-			Requires: &cli.Requires{},
-			Action:   generalVersion,
-		},
-	}...)
-}
-
-func generalID(c *cli.Context) {
-	fmt.Println(getMyID(c))
-}
-
-func generalStatus(c *cli.Context) {
-	response := httpGet(c, "system/config/insync")
-	var status struct{ ConfigInSync bool }
-	json.Unmarshal(responseToBArray(response), &status)
-	if !status.ConfigInSync {
-		die("Config out of sync")
-	}
-	fmt.Println("Config in sync")
-}
-
-func generalVersion(c *cli.Context) {
-	response := httpGet(c, "system/version")
-	version := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &version)
-	prettyPrintJSON(version)
-}

cmd/stcli/cmd_gui.go

@@ -1,127 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "gui",
-		HideHelp: true,
-		Usage:    "GUI command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "dump",
-				Usage:    "Show all GUI configuration settings",
-				Requires: &cli.Requires{},
-				Action:   guiDump,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a GUI configuration setting",
-				Requires: &cli.Requires{"setting"},
-				Action:   guiGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a GUI configuration setting",
-				Requires: &cli.Requires{"setting", "value"},
-				Action:   guiSet,
-			},
-			{
-				Name:     "unset",
-				Usage:    "Unset a GUI configuration setting",
-				Requires: &cli.Requires{"setting"},
-				Action:   guiUnset,
-			},
-		},
-	})
-}
-
-func guiDump(c *cli.Context) {
-	cfg := getConfig(c).GUI
-	writer := newTableWriter()
-	fmt.Fprintln(writer, "Enabled:\t", cfg.Enabled, "\t(enabled)")
-	fmt.Fprintln(writer, "Use HTTPS:\t", cfg.UseTLS(), "\t(tls)")
-	fmt.Fprintln(writer, "Listen Addresses:\t", cfg.Address(), "\t(address)")
-	if cfg.User != "" {
-		fmt.Fprintln(writer, "Authentication User:\t", cfg.User, "\t(username)")
-		fmt.Fprintln(writer, "Authentication Password:\t", cfg.Password, "\t(password)")
-	}
-	if cfg.APIKey != "" {
-		fmt.Fprintln(writer, "API Key:\t", cfg.APIKey, "\t(apikey)")
-	}
-	writer.Flush()
-}
-
-func guiGet(c *cli.Context) {
-	cfg := getConfig(c).GUI
-	arg := c.Args()[0]
-	switch strings.ToLower(arg) {
-	case "enabled":
-		fmt.Println(cfg.Enabled)
-	case "tls":
-		fmt.Println(cfg.UseTLS())
-	case "address":
-		fmt.Println(cfg.Address())
-	case "user":
-		if cfg.User != "" {
-			fmt.Println(cfg.User)
-		}
-	case "password":
-		if cfg.User != "" {
-			fmt.Println(cfg.Password)
-		}
-	case "apikey":
-		if cfg.APIKey != "" {
-			fmt.Println(cfg.APIKey)
-		}
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
-	}
-}
-
-func guiSet(c *cli.Context) {
-	cfg := getConfig(c)
-	arg := c.Args()[0]
-	val := c.Args()[1]
-	switch strings.ToLower(arg) {
-	case "enabled":
-		cfg.GUI.Enabled = parseBool(val)
-	case "tls":
-		cfg.GUI.RawUseTLS = parseBool(val)
-	case "address":
-		validAddress(val)
-		cfg.GUI.RawAddress = val
-	case "user":
-		cfg.GUI.User = val
-	case "password":
-		cfg.GUI.Password = val
-	case "apikey":
-		cfg.GUI.APIKey = val
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
-	}
-	setConfig(c, cfg)
-}
-
-func guiUnset(c *cli.Context) {
-	cfg := getConfig(c)
-	arg := c.Args()[0]
-	switch strings.ToLower(arg) {
-	case "user":
-		cfg.GUI.User = ""
-	case "password":
-		cfg.GUI.Password = ""
-	case "apikey":
-		cfg.GUI.APIKey = ""
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: user, password, apikey")
-	}
-	setConfig(c, cfg)
-}

cmd/stcli/cmd_options.go

@@ -1,173 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "options",
-		HideHelp: true,
-		Usage:    "Options command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "dump",
-				Usage:    "Show all Syncthing option settings",
-				Requires: &cli.Requires{},
-				Action:   optionsDump,
-			},
-			{
-				Name:     "get",
-				Usage:    "Get a Syncthing option setting",
-				Requires: &cli.Requires{"setting"},
-				Action:   optionsGet,
-			},
-			{
-				Name:     "set",
-				Usage:    "Set a Syncthing option setting",
-				Requires: &cli.Requires{"setting", "value..."},
-				Action:   optionsSet,
-			},
-		},
-	})
-}
-
-func optionsDump(c *cli.Context) {
-	cfg := getConfig(c).Options
-	writer := newTableWriter()
-
-	fmt.Fprintln(writer, "Sync protocol listen addresses:\t", strings.Join(cfg.ListenAddresses, " "), "\t(addresses)")
-	fmt.Fprintln(writer, "Global discovery enabled:\t", cfg.GlobalAnnEnabled, "\t(globalannenabled)")
-	fmt.Fprintln(writer, "Global discovery servers:\t", strings.Join(cfg.GlobalAnnServers, " "), "\t(globalannserver)")
-
-	fmt.Fprintln(writer, "Local discovery enabled:\t", cfg.LocalAnnEnabled, "\t(localannenabled)")
-	fmt.Fprintln(writer, "Local discovery port:\t", cfg.LocalAnnPort, "\t(localannport)")
-
-	fmt.Fprintln(writer, "Outgoing rate limit in KiB/s:\t", cfg.MaxSendKbps, "\t(maxsend)")
-	fmt.Fprintln(writer, "Incoming rate limit in KiB/s:\t", cfg.MaxRecvKbps, "\t(maxrecv)")
-	fmt.Fprintln(writer, "Reconnect interval in seconds:\t", cfg.ReconnectIntervalS, "\t(reconnect)")
-	fmt.Fprintln(writer, "Start browser:\t", cfg.StartBrowser, "\t(browser)")
-	fmt.Fprintln(writer, "Enable UPnP:\t", cfg.NATEnabled, "\t(nat)")
-	fmt.Fprintln(writer, "UPnP Lease in minutes:\t", cfg.NATLeaseM, "\t(natlease)")
-	fmt.Fprintln(writer, "UPnP Renewal period in minutes:\t", cfg.NATRenewalM, "\t(natrenew)")
-	fmt.Fprintln(writer, "Restart on Wake Up:\t", cfg.RestartOnWakeup, "\t(wake)")
-
-	reporting := "unrecognized value"
-	switch cfg.URAccepted {
-	case -1:
-		reporting = "false"
-	case 0:
-		reporting = "undecided/false"
-	case 1:
-		reporting = "true"
-	}
-	fmt.Fprintln(writer, "Anonymous usage reporting:\t", reporting, "\t(reporting)")
-
-	writer.Flush()
-}
-
-func optionsGet(c *cli.Context) {
-	cfg := getConfig(c).Options
-	arg := c.Args()[0]
-	switch strings.ToLower(arg) {
-	case "address":
-		fmt.Println(strings.Join(cfg.ListenAddresses, "\n"))
-	case "globalannenabled":
-		fmt.Println(cfg.GlobalAnnEnabled)
-	case "globalannservers":
-		fmt.Println(strings.Join(cfg.GlobalAnnServers, "\n"))
-	case "localannenabled":
-		fmt.Println(cfg.LocalAnnEnabled)
-	case "localannport":
-		fmt.Println(cfg.LocalAnnPort)
-	case "maxsend":
-		fmt.Println(cfg.MaxSendKbps)
-	case "maxrecv":
-		fmt.Println(cfg.MaxRecvKbps)
-	case "reconnect":
-		fmt.Println(cfg.ReconnectIntervalS)
-	case "browser":
-		fmt.Println(cfg.StartBrowser)
-	case "nat":
-		fmt.Println(cfg.NATEnabled)
-	case "natlease":
-		fmt.Println(cfg.NATLeaseM)
-	case "natrenew":
-		fmt.Println(cfg.NATRenewalM)
-	case "reporting":
-		switch cfg.URAccepted {
-		case -1:
-			fmt.Println("false")
-		case 0:
-			fmt.Println("undecided/false")
-		case 1:
-			fmt.Println("true")
-		default:
-			fmt.Println("unknown")
-		}
-	case "wake":
-		fmt.Println(cfg.RestartOnWakeup)
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
-	}
-}
-
-func optionsSet(c *cli.Context) {
-	config := getConfig(c)
-	arg := c.Args()[0]
-	val := c.Args()[1]
-	switch strings.ToLower(arg) {
-	case "address":
-		for _, item := range c.Args().Tail() {
-			validAddress(item)
-		}
-		config.Options.ListenAddresses = c.Args().Tail()
-	case "globalannenabled":
-		config.Options.GlobalAnnEnabled = parseBool(val)
-	case "globalannserver":
-		for _, item := range c.Args().Tail() {
-			validAddress(item)
-		}
-		config.Options.GlobalAnnServers = c.Args().Tail()
-	case "localannenabled":
-		config.Options.LocalAnnEnabled = parseBool(val)
-	case "localannport":
-		config.Options.LocalAnnPort = parsePort(val)
-	case "maxsend":
-		config.Options.MaxSendKbps = parseUint(val)
-	case "maxrecv":
-		config.Options.MaxRecvKbps = parseUint(val)
-	case "reconnect":
-		config.Options.ReconnectIntervalS = parseUint(val)
-	case "browser":
-		config.Options.StartBrowser = parseBool(val)
-	case "nat":
-		config.Options.NATEnabled = parseBool(val)
-	case "natlease":
-		config.Options.NATLeaseM = parseUint(val)
-	case "natrenew":
-		config.Options.NATRenewalM = parseUint(val)
-	case "reporting":
-		switch strings.ToLower(val) {
-		case "u", "undecided", "unset":
-			config.Options.URAccepted = 0
-		default:
-			boolvalue := parseBool(val)
-			if boolvalue {
-				config.Options.URAccepted = 1
-			} else {
-				config.Options.URAccepted = -1
-			}
-		}
-	case "wake":
-		config.Options.RestartOnWakeup = parseBool(val)
-	default:
-		die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
-	}
-	setConfig(c, config)
-}

cmd/stcli/cmd_report.go

@@ -1,72 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-func init() {
-	cliCommands = append(cliCommands, cli.Command{
-		Name:     "report",
-		HideHelp: true,
-		Usage:    "Reporting command group",
-		Subcommands: []cli.Command{
-			{
-				Name:     "system",
-				Usage:    "Report system state",
-				Requires: &cli.Requires{},
-				Action:   reportSystem,
-			},
-			{
-				Name:     "connections",
-				Usage:    "Report about connections to other devices",
-				Requires: &cli.Requires{},
-				Action:   reportConnections,
-			},
-			{
-				Name:     "usage",
-				Usage:    "Usage report",
-				Requires: &cli.Requires{},
-				Action:   reportUsage,
-			},
-		},
-	})
-}
-
-func reportSystem(c *cli.Context) {
-	response := httpGet(c, "system/status")
-	data := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &data)
-	prettyPrintJSON(data)
-}
-
-func reportConnections(c *cli.Context) {
-	response := httpGet(c, "system/connections")
-	data := make(map[string]map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &data)
-	var overall map[string]interface{}
-	for key, value := range data {
-		if key == "total" {
-			overall = value
-			continue
-		}
-		value["Device ID"] = key
-		prettyPrintJSON(value)
-		fmt.Println()
-	}
-	if overall != nil {
-		fmt.Println("=== Overall statistics ===")
-		prettyPrintJSON(overall)
-	}
-}
-
-func reportUsage(c *cli.Context) {
-	response := httpGet(c, "svc/report")
-	report := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &report)
-	prettyPrintJSON(report)
-}

cmd/stcli/labels.go

@@ -1,31 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-var jsonAttributeLabels = map[string]string{
-	"folderMaxMiB":   "Largest folder size in MiB",
-	"folderMaxFiles": "Largest folder file count",
-	"longVersion":    "Long version",
-	"totMiB":         "Total size in MiB",
-	"totFiles":       "Total files",
-	"uniqueID":       "Unique ID",
-	"numFolders":     "Folder count",
-	"numDevices":     "Device count",
-	"memoryUsageMiB": "Memory usage in MiB",
-	"memorySize":     "Total memory in MiB",
-	"sha256Perf":     "SHA256 Benchmark",
-	"At":             "Last contacted",
-	"Completion":     "Percent complete",
-	"InBytesTotal":   "Total bytes received",
-	"OutBytesTotal":  "Total bytes sent",
-	"ClientVersion":  "Client version",
-	"alloc":          "Memory allocated in bytes",
-	"sys":            "Memory using in bytes",
-	"cpuPercent":     "CPU load in percent",
-	"extAnnounceOK":  "External announcments working",
-	"goroutines":     "Number of Go routines",
-	"myID":           "Client ID",
-	"tilde":          "Tilde expands to",
-	"arch":           "Architecture",
-	"os":             "OS",
-}

cmd/stcli/main.go

@@ -1,63 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"sort"
-
-	"github.com/AudriusButkevicius/cli"
-)
-
-type ByAlphabet []cli.Command
-
-func (a ByAlphabet) Len() int           { return len(a) }
-func (a ByAlphabet) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
-func (a ByAlphabet) Less(i, j int) bool { return a[i].Name < a[j].Name }
-
-var cliCommands []cli.Command
-
-func main() {
-	app := cli.NewApp()
-	app.Name = "syncthing-cli"
-	app.Author = "Audrius Butkevičius"
-	app.Email = "audrius.butkevicius@gmail.com"
-	app.Usage = "Syncthing command line interface"
-	app.Version = "0.1"
-	app.HideHelp = true
-
-	app.Flags = []cli.Flag{
-		cli.StringFlag{
-			Name:   "endpoint, e",
-			Value:  "http://127.0.0.1:8384",
-			Usage:  "End point to connect to",
-			EnvVar: "STENDPOINT",
-		},
-		cli.StringFlag{
-			Name:   "apikey, k",
-			Value:  "",
-			Usage:  "API Key",
-			EnvVar: "STAPIKEY",
-		},
-		cli.StringFlag{
-			Name:   "username, u",
-			Value:  "",
-			Usage:  "Username",
-			EnvVar: "STUSERNAME",
-		},
-		cli.StringFlag{
-			Name:   "password, p",
-			Value:  "",
-			Usage:  "Password",
-			EnvVar: "STPASSWORD",
-		},
-		cli.BoolFlag{
-			Name:   "insecure, i",
-			Usage:  "Do not verify SSL certificate",
-			EnvVar: "STINSECURE",
-		},
-	}
-
-	sort.Sort(ByAlphabet(cliCommands))
-	app.Commands = cliCommands
-	app.RunAndExitOnError()
-}

cmd/stcli/utils.go

@@ -1,165 +0,0 @@
-// Copyright (C) 2014 Audrius Butkevičius
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"text/tabwriter"
-	"unicode"
-
-	"github.com/AudriusButkevicius/cli"
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-func responseToBArray(response *http.Response) []byte {
-	defer response.Body.Close()
-	bytes, err := ioutil.ReadAll(response.Body)
-	if err != nil {
-		die(err)
-	}
-	return bytes
-}
-
-func die(vals ...interface{}) {
-	if len(vals) > 1 || vals[0] != nil {
-		os.Stderr.WriteString(fmt.Sprintln(vals...))
-		os.Exit(1)
-	}
-}
-
-func wrappedHTTPPost(url string) func(c *cli.Context) {
-	return func(c *cli.Context) {
-		httpPost(c, url, "")
-	}
-}
-
-func prettyPrintJSON(json map[string]interface{}) {
-	writer := newTableWriter()
-	remap := make(map[string]interface{})
-	for k, v := range json {
-		key, ok := jsonAttributeLabels[k]
-		if !ok {
-			key = firstUpper(k)
-		}
-		remap[key] = v
-	}
-
-	jsonKeys := make([]string, 0, len(remap))
-	for key := range remap {
-		jsonKeys = append(jsonKeys, key)
-	}
-	sort.Strings(jsonKeys)
-	for _, k := range jsonKeys {
-		var value string
-		rvalue := remap[k]
-		switch rvalue.(type) {
-		case int, int16, int32, int64, uint, uint16, uint32, uint64, float32, float64:
-			value = fmt.Sprintf("%.0f", rvalue)
-		default:
-			value = fmt.Sprint(rvalue)
-		}
-		if value == "" {
-			continue
-		}
-		fmt.Fprintln(writer, k+":\t"+value)
-	}
-	writer.Flush()
-}
-
-func firstUpper(str string) string {
-	for i, v := range str {
-		return string(unicode.ToUpper(v)) + str[i+1:]
-	}
-	return ""
-}
-
-func newTableWriter() *tabwriter.Writer {
-	writer := new(tabwriter.Writer)
-	writer.Init(os.Stdout, 0, 8, 0, '\t', 0)
-	return writer
-}
-
-func getMyID(c *cli.Context) string {
-	response := httpGet(c, "system/status")
-	data := make(map[string]interface{})
-	json.Unmarshal(responseToBArray(response), &data)
-	return data["myID"].(string)
-}
-
-func getConfig(c *cli.Context) config.Configuration {
-	response := httpGet(c, "system/config")
-	config := config.Configuration{}
-	json.Unmarshal(responseToBArray(response), &config)
-	return config
-}
-
-func setConfig(c *cli.Context, cfg config.Configuration) {
-	body, err := json.Marshal(cfg)
-	die(err)
-	response := httpPost(c, "system/config", string(body))
-	if response.StatusCode != 200 {
-		die("Unexpected status code", response.StatusCode)
-	}
-}
-
-func parseBool(input string) bool {
-	val, err := strconv.ParseBool(input)
-	if err != nil {
-		die(input + " is not a valid value for a boolean")
-	}
-	return val
-}
-
-func parseInt(input string) int {
-	val, err := strconv.ParseInt(input, 0, 64)
-	if err != nil {
-		die(input + " is not a valid value for an integer")
-	}
-	return int(val)
-}
-
-func parseUint(input string) int {
-	val, err := strconv.ParseUint(input, 0, 64)
-	if err != nil {
-		die(input + " is not a valid value for an unsigned integer")
-	}
-	return int(val)
-}
-
-func parsePort(input string) int {
-	port := parseUint(input)
-	if port < 1 || port > 65535 {
-		die(input + " is not a valid port\nExpected value between 1 and 65535")
-	}
-	return port
-}
-
-func validAddress(input string) {
-	tokens := strings.Split(input, ":")
-	if len(tokens) != 2 {
-		die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
-	}
-	matched, err := regexp.MatchString("^[a-zA-Z0-9]+([-a-zA-Z0-9.]+[-a-zA-Z0-9]+)?$", tokens[0])
-	die(err)
-	if !matched {
-		die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
-	}
-	parsePort(tokens[1])
-}
-
-func parseDeviceID(input string) protocol.DeviceID {
-	device, err := protocol.DeviceIDFromString(input)
-	if err != nil {
-		die(input + " is not a valid device id")
-	}
-	return device
-}

cmd/stcompdirs/main.go

@@ -7,7 +7,6 @@
 package main
 
 import (
-	"crypto/md5"
 	"errors"
 	"flag"
 	"fmt"
@@ -15,6 +14,8 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 func main() {
@@ -60,7 +61,7 @@ func compareDirectories(dirs ...string) error {
 				} else if res[i].name > res[0].name {
 					return fmt.Errorf("%s missing %v (present in %s)", dirs[i], res[0], dirs[0])
 				}
-				return fmt.Errorf("Mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
+				return fmt.Errorf("mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
 			}
 		}
 
@@ -74,7 +75,7 @@ type fileInfo struct {
 	name string
 	mode os.FileMode
 	mod  int64
-	hash [16]byte
+	hash [sha256.Size]byte
 }
 
 func (f fileInfo) String() string {
@@ -106,11 +107,7 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 			if err != nil {
 				return err
 			}
-			h := md5.New()
-			h.Write([]byte(tgt))
-			hash := h.Sum(nil)
-
-			copy(f.hash[:], hash)
+			f.hash = sha256.Sum256([]byte(tgt))
 		} else if info.IsDir() {
 			f = fileInfo{
 				name: rn,
@@ -123,7 +120,7 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 				mode: info.Mode(),
 				mod:  info.ModTime().Unix(),
 			}
-			sum, err := md5file(path)
+			sum, err := sha256file(path)
 			if err != nil {
 				return err
 			}
@@ -150,14 +147,14 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 	return errc
 }
 
-func md5file(fname string) (hash [16]byte, err error) {
+func sha256file(fname string) (hash [sha256.Size]byte, err error) {
 	f, err := os.Open(fname)
 	if err != nil {
 		return
 	}
 	defer f.Close()
 
-	h := md5.New()
+	h := sha256.New()
 	io.Copy(h, f)
 	hb := h.Sum(nil)
 	copy(hash[:], hb)

cmd/stcrashreceiver/main.go

@@ -0,0 +1,125 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Command stcrashreceiver is a trivial HTTP server that allows two things:
+//
+// - uploading files (crash reports) named like a SHA256 hash using a PUT request
+// - checking whether such file exists using a HEAD request
+//
+// Typically this should be deployed behind something that manages HTTPS.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/syncthing/syncthing/lib/sha256"
+	"github.com/syncthing/syncthing/lib/ur"
+
+	raven "github.com/getsentry/raven-go"
+)
+
+const maxRequestSize = 1 << 20 // 1 MiB
+
+func main() {
+	dir := flag.String("dir", ".", "Parent directory to store crash and failure reports in")
+	dsn := flag.String("dsn", "", "Sentry DSN")
+	listen := flag.String("listen", ":22039", "HTTP listen address")
+	flag.Parse()
+
+	mux := http.NewServeMux()
+
+	cr := &crashReceiver{
+		dir: filepath.Join(*dir, "crash_reports"),
+		dsn: *dsn,
+	}
+	mux.Handle("/", cr)
+
+	if *dsn != "" {
+		mux.HandleFunc("/newcrash/failure", handleFailureFn(*dsn, filepath.Join(*dir, "failure_reports")))
+	}
+
+	log.SetOutput(os.Stdout)
+	if err := http.ListenAndServe(*listen, mux); err != nil {
+		log.Fatalln("HTTP serve:", err)
+	}
+}
+
+func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *http.Request) {
+	return func(w http.ResponseWriter, req *http.Request) {
+		lr := io.LimitReader(req.Body, maxRequestSize)
+		bs, err := io.ReadAll(lr)
+		req.Body.Close()
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+
+		var reports []ur.FailureReport
+		err = json.Unmarshal(bs, &reports)
+		if err != nil {
+			http.Error(w, err.Error(), 400)
+			return
+		}
+		if len(reports) == 0 {
+			// Shouldn't happen
+			log.Printf("Got zero failure reports")
+			return
+		}
+
+		version, err := parseVersion(reports[0].Version)
+		if err != nil {
+			http.Error(w, err.Error(), 400)
+			return
+		}
+		for _, r := range reports {
+			pkt := packet(version, "failure")
+			pkt.Message = r.Description
+			pkt.Extra = raven.Extra{
+				"count": r.Count,
+			}
+			for k, v := range r.Extra {
+				pkt.Extra[k] = v
+			}
+			if r.Goroutines != "" {
+				url, err := saveFailureWithGoroutines(r.FailureData, failureDir)
+				if err != nil {
+					log.Println("Saving failure report:", err)
+					http.Error(w, "Internal server error", http.StatusInternalServerError)
+					return
+				}
+				pkt.Extra["goroutinesURL"] = url
+			}
+			message := sanitizeMessageLDB(r.Description)
+			pkt.Fingerprint = []string{message}
+
+			if err := sendReport(dsn, pkt, userIDFor(req)); err != nil {
+				log.Println("Failed to send failure report:", err)
+			} else {
+				log.Println("Sent failure report:", r.Description)
+			}
+		}
+	}
+}
+
+func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string, error) {
+	bs := make([]byte, len(data.Description)+len(data.Goroutines))
+	copy(bs, data.Description)
+	copy(bs[len(data.Description):], data.Goroutines)
+	id := fmt.Sprintf("%x", sha256.Sum256(bs))
+	path := fullPathCompressed(failureDir, id)
+	err := compressAndWrite(bs, path)
+	if err != nil {
+		return "", err
+	}
+	return reportServer + path, nil
+}

cmd/stcrashreceiver/sentry.go

@@ -0,0 +1,260 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"regexp"
+	"strings"
+	"sync"
+
+	raven "github.com/getsentry/raven-go"
+	"github.com/maruel/panicparse/stack"
+)
+
+const reportServer = "https://crash.syncthing.net/report/"
+
+var loader = newGithubSourceCodeLoader()
+
+func init() {
+	raven.SetSourceCodeLoader(loader)
+}
+
+var (
+	clients    = make(map[string]*raven.Client)
+	clientsMut sync.Mutex
+)
+
+func sendReport(dsn string, pkt *raven.Packet, userID string) error {
+	pkt.Interfaces = append(pkt.Interfaces, &raven.User{ID: userID})
+
+	clientsMut.Lock()
+	defer clientsMut.Unlock()
+
+	cli, ok := clients[dsn]
+	if !ok {
+		var err error
+		cli, err = raven.New(dsn)
+		if err != nil {
+			return err
+		}
+		clients[dsn] = cli
+	}
+
+	// The client sets release and such on the packet before sending, in the
+	// misguided idea that it knows this better than than the packet we give
+	// it. So we copy the values from the packet to the client first...
+	cli.SetRelease(pkt.Release)
+	cli.SetEnvironment(pkt.Environment)
+
+	defer cli.Wait()
+	_, errC := cli.Capture(pkt, nil)
+	return <-errC
+}
+
+func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
+	parts := bytes.SplitN(report, []byte("\n"), 2)
+	if len(parts) != 2 {
+		return nil, errors.New("no first line")
+	}
+
+	version, err := parseVersion(string(parts[0]))
+	if err != nil {
+		return nil, err
+	}
+	report = parts[1]
+
+	foundPanic := false
+	var subjectLine []byte
+	for {
+		parts = bytes.SplitN(report, []byte("\n"), 2)
+		if len(parts) != 2 {
+			return nil, errors.New("no panic line found")
+		}
+
+		line := parts[0]
+		report = parts[1]
+
+		if foundPanic {
+			// The previous line was our "Panic at ..." header. We are now
+			// at the beginning of the real panic trace and this is our
+			// subject line.
+			subjectLine = line
+			break
+		} else if bytes.HasPrefix(line, []byte("Panic at")) {
+			foundPanic = true
+		}
+	}
+
+	r := bytes.NewReader(report)
+	ctx, err := stack.ParseDump(r, io.Discard, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// Lock the source code loader to the version we are processing here.
+	if version.commit != "" {
+		// We have a commit hash, so we know exactly which source to use
+		loader.LockWithVersion(version.commit)
+	} else if strings.HasPrefix(version.tag, "v") {
+		// Lets hope the tag is close enough
+		loader.LockWithVersion(version.tag)
+	} else {
+		// Last resort
+		loader.LockWithVersion("main")
+	}
+	defer loader.Unlock()
+
+	var trace raven.Stacktrace
+	for _, gr := range ctx.Goroutines {
+		if gr.First {
+			trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
+			for i, sc := range gr.Stack.Calls {
+				trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name(), sc.SrcPath, sc.Line, 3, nil)
+			}
+			break
+		}
+	}
+
+	pkt := packet(version, "crash")
+	pkt.Message = string(subjectLine)
+	pkt.Extra = raven.Extra{
+		"url": reportServer + path,
+	}
+	pkt.Interfaces = []raven.Interface{&trace}
+	pkt.Fingerprint = crashReportFingerprint(pkt.Message)
+
+	return pkt, nil
+}
+
+var (
+	indexRe          = regexp.MustCompile(`\[[-:0-9]+\]`)
+	sizeRe           = regexp.MustCompile(`(length|capacity) [0-9]+`)
+	ldbPosRe         = regexp.MustCompile(`(\(pos=)([0-9]+)\)`)
+	ldbChecksumRe    = regexp.MustCompile(`(want=0x)([a-z0-9]+)( got=0x)([a-z0-9]+)`)
+	ldbFileRe        = regexp.MustCompile(`(\[file=)([0-9]+)(\.ldb\])`)
+	ldbInternalKeyRe = regexp.MustCompile(`(internal key ")[^"]+(", len=)[0-9]+`)
+	ldbPathRe        = regexp.MustCompile(`(open|write|read) .+[\\/].+[\\/]index[^\\/]+[\\/][^\\/]+: `)
+)
+
+func sanitizeMessageLDB(message string) string {
+	message = ldbPosRe.ReplaceAllString(message, "${1}x)")
+	message = ldbFileRe.ReplaceAllString(message, "${1}x${3}")
+	message = ldbChecksumRe.ReplaceAllString(message, "${1}X${3}X")
+	message = ldbInternalKeyRe.ReplaceAllString(message, "${1}x${2}x")
+	message = ldbPathRe.ReplaceAllString(message, "$1 x: ")
+	return message
+}
+
+func crashReportFingerprint(message string) []string {
+	// Do not fingerprint on the stack in case of db corruption or fatal
+	// db io error - where it occurs doesn't matter.
+	orig := message
+	message = sanitizeMessageLDB(message)
+	if message != orig {
+		return []string{message}
+	}
+
+	message = indexRe.ReplaceAllString(message, "[x]")
+	message = sizeRe.ReplaceAllString(message, "$1 x")
+
+	// {{ default }} is what sentry uses as a fingerprint by default. While
+	// never specified, the docs point at this being some hash derived from the
+	// stack trace. Here we include the filtered panic message on top of that.
+	// https://docs.sentry.io/platforms/go/data-management/event-grouping/sdk-fingerprinting/#basic-example
+	return []string{"{{ default }}", message}
+}
+
+// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]
+var longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
+
+type version struct {
+	version  string   // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
+	tag      string   // "v1.1.4-rc.1"
+	commit   string   // "6aaae618", blank when absent
+	codename string   // "Erbium Earthworm"
+	runtime  string   // "go1.12.5"
+	goos     string   // "darwin"
+	goarch   string   // "amd64"
+	builder  string   // "jb@kvin.kastelo.net"
+	extra    []string // "foo", "bar"
+}
+
+func (v version) environment() string {
+	if v.commit != "" {
+		return "Development"
+	}
+	if strings.Contains(v.tag, "-rc.") {
+		return "Candidate"
+	}
+	if strings.Contains(v.tag, "-") {
+		return "Beta"
+	}
+	return "Stable"
+}
+
+func parseVersion(line string) (version, error) {
+	m := longVersionRE.FindStringSubmatch(line)
+	if len(m) == 0 {
+		return version{}, errors.New("unintelligeble version string")
+	}
+
+	v := version{
+		version:  m[1],
+		codename: m[2],
+		runtime:  m[3],
+		goos:     m[4],
+		goarch:   m[5],
+		builder:  m[6],
+	}
+
+	parts := strings.Split(v.version, "+")
+	v.tag = parts[0]
+	if len(parts) > 1 {
+		fields := strings.Split(parts[1], "-")
+		if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
+			v.commit = fields[1][1:]
+		}
+	}
+
+	if len(m) >= 8 && m[7] != "" {
+		tags := strings.Split(m[7], ",")
+		for i := range tags {
+			tags[i] = strings.TrimSpace(tags[i])
+		}
+		v.extra = tags
+	}
+
+	return v, nil
+}
+
+func packet(version version, reportType string) *raven.Packet {
+	pkt := &raven.Packet{
+		Platform:    "go",
+		Release:     version.tag,
+		Environment: version.environment(),
+		Tags: raven.Tags{
+			raven.Tag{Key: "version", Value: version.version},
+			raven.Tag{Key: "tag", Value: version.tag},
+			raven.Tag{Key: "codename", Value: version.codename},
+			raven.Tag{Key: "runtime", Value: version.runtime},
+			raven.Tag{Key: "goos", Value: version.goos},
+			raven.Tag{Key: "goarch", Value: version.goarch},
+			raven.Tag{Key: "builder", Value: version.builder},
+			raven.Tag{Key: "report_type", Value: reportType},
+		},
+	}
+	if version.commit != "" {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
+	}
+	for _, tag := range version.extra {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: tag, Value: "1"})
+	}
+	return pkt
+}

cmd/stcrashreceiver/sentry_test.go

@@ -0,0 +1,141 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+func TestParseVersion(t *testing.T) {
+	cases := []struct {
+		longVersion string
+		parsed      version
+	}{
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+			},
+		},
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+				extra:    []string{"foo", "bar"},
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		v, err := parseVersion(tc.longVersion)
+		if err != nil {
+			t.Errorf("%s\nerror: %v\n", tc.longVersion, err)
+			continue
+		}
+		if fmt.Sprint(v) != fmt.Sprint(tc.parsed) {
+			t.Errorf("%s\nA: %v\nE: %v\n", tc.longVersion, v, tc.parsed)
+		}
+	}
+}
+
+func TestParseReport(t *testing.T) {
+	bs, err := os.ReadFile("_testdata/panic.log")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pkt, err := parseCrashReport("1/2/345", bs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bs, err = pkt.JSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Printf("%s\n", bs)
+}
+
+func TestCrashReportFingerprint(t *testing.T) {
+	cases := []struct {
+		message, exp string
+		ldb          bool
+	}{
+		{
+			message: "panic: leveldb/table: corruption on data-block (pos=51308946): checksum mismatch, want=0xa89f9aa0 got=0xd27cc4c7 [file=004003.ldb]",
+			exp:     "panic: leveldb/table: corruption on data-block (pos=x): checksum mismatch, want=0xX got=0xX [file=x.ldb]",
+			ldb:     true,
+		},
+		{
+			message: "panic: leveldb/table: corruption on table-footer (pos=248): bad magic number [file=001370.ldb]",
+			exp:     "panic: leveldb/table: corruption on table-footer (pos=x): bad magic number [file=x.ldb]",
+			ldb:     true,
+		},
+		{
+			message: "panic: runtime error: slice bounds out of range [4294967283:4194304]",
+			exp:     "panic: runtime error: slice bounds out of range [x]",
+		},
+		{
+			message: "panic: runtime error: slice bounds out of range [-2:]",
+			exp:     "panic: runtime error: slice bounds out of range [x]",
+		},
+		{
+			message: "panic: runtime error: slice bounds out of range [:4294967283] with capacity 32768",
+			exp:     "panic: runtime error: slice bounds out of range [x] with capacity x",
+		},
+		{
+			message: "panic: runtime error: index out of range [0] with length 0",
+			exp:     "panic: runtime error: index out of range [x] with length x",
+		},
+		{
+			message: `panic: leveldb: internal key "\x01", len=1: invalid length`,
+			exp:     `panic: leveldb: internal key "x", len=x: invalid length`,
+			ldb:     true,
+		},
+		{
+			message: `panic: write /var/syncthing/config/index-v0.14.0.db/2732813.log: cannot allocate memory`,
+			exp:     `panic: write x: cannot allocate memory`,
+			ldb:     true,
+		},
+		{
+			message: `panic: filling Blocks: read C:\Users\Serv-Resp-Tizayuca\AppData\Local\Syncthing\index-v0.14.0.db\006561.ldb: Error de datos (comprobación de redundancia cíclica).`,
+			exp:     `panic: filling Blocks: read x: Error de datos (comprobación de redundancia cíclica).`,
+			ldb:     true,
+		},
+	}
+
+	for i, tc := range cases {
+		fingerprint := crashReportFingerprint(tc.message)
+
+		expLen := 2
+		if tc.ldb {
+			expLen = 1
+		}
+		if l := len(fingerprint); l != expLen {
+			t.Errorf("tc %v: Unexpected fingerprint length: %v != %v", i, l, expLen)
+		} else if msg := fingerprint[expLen-1]; msg != tc.exp {
+			t.Errorf("tc %v:\n\"%v\" !=\n\"%v\"", i, msg, tc.exp)
+		}
+	}
+}

cmd/stcrashreceiver/sourcecodeloader.go

@@ -0,0 +1,118 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	urlPrefix   = "https://raw.githubusercontent.com/syncthing/syncthing/"
+	httpTimeout = 10 * time.Second
+)
+
+type githubSourceCodeLoader struct {
+	mut     sync.Mutex
+	version string
+	cache   map[string]map[string][][]byte // version -> file -> lines
+	client  *http.Client
+}
+
+func newGithubSourceCodeLoader() *githubSourceCodeLoader {
+	return &githubSourceCodeLoader{
+		cache:  make(map[string]map[string][][]byte),
+		client: &http.Client{Timeout: httpTimeout},
+	}
+}
+
+func (l *githubSourceCodeLoader) LockWithVersion(version string) {
+	l.mut.Lock()
+	l.version = version
+	if _, ok := l.cache[version]; !ok {
+		l.cache[version] = make(map[string][][]byte)
+	}
+}
+
+func (l *githubSourceCodeLoader) Unlock() {
+	l.mut.Unlock()
+}
+
+func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]byte, int) {
+	filename = filepath.ToSlash(filename)
+	lines, ok := l.cache[l.version][filename]
+	if !ok {
+		// Cache whatever we managed to find (or nil if nothing, so we don't try again)
+		defer func() {
+			l.cache[l.version][filename] = lines
+		}()
+
+		knownPrefixes := []string{"/lib/", "/cmd/"}
+		var idx int
+		for _, pref := range knownPrefixes {
+			idx = strings.Index(filename, pref)
+			if idx >= 0 {
+				break
+			}
+		}
+		if idx == -1 {
+			return nil, 0
+		}
+
+		url := urlPrefix + l.version + filename[idx:]
+		resp, err := l.client.Get(url)
+
+		if err != nil {
+			fmt.Println("Loading source:", err)
+			return nil, 0
+		}
+		if resp.StatusCode != http.StatusOK {
+			fmt.Println("Loading source:", resp.Status)
+			return nil, 0
+		}
+		data, err := io.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		if err != nil {
+			fmt.Println("Loading source:", err.Error())
+			return nil, 0
+		}
+		lines = bytes.Split(data, []byte{'\n'})
+	}
+
+	return getLineFromLines(lines, line, context)
+}
+
+func getLineFromLines(lines [][]byte, line, context int) ([][]byte, int) {
+	if lines == nil {
+		// cached error from ReadFile: return no lines
+		return nil, 0
+	}
+
+	line-- // stack trace lines are 1-indexed
+	start := line - context
+	var idx int
+	if start < 0 {
+		start = 0
+		idx = line
+	} else {
+		idx = context
+	}
+	end := line + context + 1
+	if line >= len(lines) {
+		return nil, 0
+	}
+	if end > len(lines) {
+		end = len(lines)
+	}
+	return lines[start:end], idx
+}

cmd/stcrashreceiver/stcrashreceiver.go

@@ -0,0 +1,141 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+type crashReceiver struct {
+	dir string
+	dsn string
+}
+
+func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	// The final path component should be a SHA256 hash in hex, so 64 hex
+	// characters. We don't care about case on the request but use lower
+	// case internally.
+	reportID := strings.ToLower(path.Base(req.URL.Path))
+	if len(reportID) != 64 {
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+	for _, c := range reportID {
+		if c >= 'a' && c <= 'f' {
+			continue
+		}
+		if c >= '0' && c <= '9' {
+			continue
+		}
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+
+	// The location of the report on disk, compressed
+	fullPath := filepath.Join(r.dir, r.dirFor(reportID), reportID) + ".gz"
+
+	switch req.Method {
+	case http.MethodGet:
+		r.serveGet(fullPath, w, req)
+	case http.MethodHead:
+		r.serveHead(fullPath, w, req)
+	case http.MethodPut:
+		r.servePut(reportID, fullPath, w, req)
+	default:
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+// serveGet responds to GET requests by serving the uncompressed report.
+func (*crashReceiver) serveGet(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	fd, err := os.Open(fullPath)
+	if err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+		return
+	}
+
+	defer fd.Close()
+	gr, err := gzip.NewReader(fd)
+	if err != nil {
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+	_, _ = io.Copy(w, gr) // best effort
+}
+
+// serveHead responds to HEAD requests by checking if the named report
+// already exists in the system.
+func (*crashReceiver) serveHead(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	if _, err := os.Lstat(fullPath); err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+	}
+}
+
+// servePut accepts and stores the given report.
+func (r *crashReceiver) servePut(reportID, fullPath string, w http.ResponseWriter, req *http.Request) {
+	// Ensure the destination directory exists
+	if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
+		log.Println("Creating directory:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Read at most maxRequestSize of report data.
+	log.Println("Receiving report", reportID)
+	lr := io.LimitReader(req.Body, maxRequestSize)
+	bs, err := io.ReadAll(lr)
+	if err != nil {
+		log.Println("Reading report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Compress the report for storage
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	_, _ = gw.Write(bs) // can't fail
+	gw.Close()
+
+	// Create an output file with the compressed report
+	err = os.WriteFile(fullPath, buf.Bytes(), 0644)
+	if err != nil {
+		log.Println("Saving report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Send the report to Sentry
+	if r.dsn != "" {
+		// Remote ID
+		user := userIDFor(req)
+
+		go func() {
+			// There's no need for the client to have to wait for this part.
+			pkt, err := parseCrashReport(reportID, bs)
+			if err != nil {
+				log.Println("Failed to parse crash report:", err)
+				return
+			}
+			if err := sendReport(r.dsn, pkt, user); err != nil {
+				log.Println("Failed to send crash report:", err)
+			}
+		}()
+	}
+}
+
+// 01234567890abcdef... => 01/23
+func (*crashReceiver) dirFor(base string) string {
+	return filepath.Join(base[0:2], base[2:4])
+}

cmd/stcrashreceiver/util.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/sha256"
+)
+
+// userIDFor returns a string we can use as the user ID for the purpose of
+// counting affected users. It's the truncated hash of a salt, the user
+// remote IP, and the current month.
+func userIDFor(req *http.Request) string {
+	addr := req.RemoteAddr
+	if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
+		addr = fwd
+	}
+	if host, _, err := net.SplitHostPort(addr); err == nil {
+		addr = host
+	}
+	now := time.Now().Format("200601")
+	salt := "stcrashreporter"
+	hash := sha256.Sum256([]byte(salt + addr + now))
+	return fmt.Sprintf("%x", hash[:8])
+}
+
+// 01234567890abcdef... => 01/23
+func dirFor(base string) string {
+	return filepath.Join(base[0:2], base[2:4])
+}
+
+func fullPathCompressed(root, reportID string) string {
+	return filepath.Join(root, dirFor(reportID), reportID) + ".gz"
+}
+
+func compressAndWrite(bs []byte, fullPath string) error {
+	// Compress the report for storage
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	_, _ = gw.Write(bs) // can't fail
+	gw.Close()
+
+	// Create an output file with the compressed report
+	return os.WriteFile(fullPath, buf.Bytes(), 0644)
+}

cmd/stdisco/main.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/binary"
 	"flag"
@@ -47,14 +48,16 @@ func main() {
 		log.Println("My ID:", myID)
 	}
 
-	runbeacon(beacon.NewMulticast(mc), fake)
-	runbeacon(beacon.NewBroadcast(bc), fake)
+	ctx := context.Background()
+
+	runbeacon(ctx, beacon.NewMulticast(mc), fake)
+	runbeacon(ctx, beacon.NewBroadcast(bc), fake)
 
 	select {}
 }
 
-func runbeacon(bc beacon.Interface, fake bool) {
-	go bc.Serve()
+func runbeacon(ctx context.Context, bc beacon.Interface, fake bool) {
+	go bc.Serve(ctx)
 	go recv(bc)
 	if fake {
 		go send(bc)

cmd/stdiscosrv/LICENSE

@@ -1,19 +0,0 @@
-Copyright (C) 2014-2015 The Discosrv Authors
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-- The above copyright notice and this permission notice shall be included in
-  all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

cmd/stdiscosrv/README.md

@@ -6,33 +6,5 @@ This is the global discovery server for the `syncthing` project.
 Usage
 -----
 
-The discovery server supports `ql` and `postgres` backends.
-Specify the backend via `-db-backend` and the database DSN via `-db-dsn`.
+https://docs.syncthing.net/users/stdiscosrv.html
 
-By default it will use in-memory `ql` backend. If you wish to persist the
-information on disk between restarts in `ql`, specify a file DSN:
-
-```bash
-$ stdiscosrv -db-dsn="file:///var/run/stdiscosrv.db"
-```
-
-For `postgres`, you will need to create a database and a user with permissions
-to create tables in it, then start the stdiscosrv as follows:
-
-```bash
-$ export STDISCOSRV_DB_DSN="postgres://user:password@localhost/databasename"
-$ stdiscosrv -db-backend="postgres"
-```
-
-You can pass the DSN as command line option, but the value what you pass in will
-be visible in most process managers, potentially exposing the database password
-to other users.
-
-In all cases, the appropriate tables and indexes will be created at first
-startup. If it doesn't exit with an error, you're fine.
-
-See `stdiscosrv -help` for other options.
-
-##### Third-party attribution
-
-[cznic/lldb](https://github.com/cznic/lldb), Copyright (C) 2014 The lldb Authors.

cmd/stdiscosrv/apisrv.go

@@ -0,0 +1,481 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"log"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/url"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+// announcement is the format received from and sent to clients
+type announcement struct {
+	Seen      time.Time `json:"seen"`
+	Addresses []string  `json:"addresses"`
+}
+
+type apiSrv struct {
+	addr     string
+	cert     tls.Certificate
+	db       database
+	listener net.Listener
+	repl     replicator // optional
+	useHTTP  bool
+
+	mapsMut sync.Mutex
+	misses  map[string]int32
+}
+
+type requestID int64
+
+func (i requestID) String() string {
+	return fmt.Sprintf("%016x", int64(i))
+}
+
+type contextKey int
+
+const idKey contextKey = iota
+
+func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP bool) *apiSrv {
+	return &apiSrv{
+		addr:    addr,
+		cert:    cert,
+		db:      db,
+		repl:    repl,
+		useHTTP: useHTTP,
+		misses:  make(map[string]int32),
+	}
+}
+
+func (s *apiSrv) Serve(_ context.Context) error {
+	if s.useHTTP {
+		listener, err := net.Listen("tcp", s.addr)
+		if err != nil {
+			log.Println("Listen:", err)
+			return err
+		}
+		s.listener = listener
+	} else {
+		tlsCfg := &tls.Config{
+			Certificates:           []tls.Certificate{s.cert},
+			ClientAuth:             tls.RequestClientCert,
+			SessionTicketsDisabled: true,
+			MinVersion:             tls.VersionTLS12,
+			CipherSuites: []uint16{
+				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+			},
+		}
+
+		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
+		if err != nil {
+			log.Println("Listen:", err)
+			return err
+		}
+		s.listener = tlsListener
+	}
+
+	http.HandleFunc("/", s.handler)
+	http.HandleFunc("/ping", handlePing)
+
+	srv := &http.Server{
+		ReadTimeout:    httpReadTimeout,
+		WriteTimeout:   httpWriteTimeout,
+		MaxHeaderBytes: httpMaxHeaderBytes,
+	}
+
+	err := srv.Serve(s.listener)
+	if err != nil {
+		log.Println("Serve:", err)
+	}
+	return err
+}
+
+var topCtx = context.Background()
+
+func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
+	t0 := time.Now()
+
+	lw := NewLoggingResponseWriter(w)
+
+	defer func() {
+		diff := time.Since(t0)
+		apiRequestsSeconds.WithLabelValues(req.Method).Observe(diff.Seconds())
+		apiRequestsTotal.WithLabelValues(req.Method, strconv.Itoa(lw.statusCode)).Inc()
+	}()
+
+	reqID := requestID(rand.Int63())
+	ctx := context.WithValue(topCtx, idKey, reqID)
+
+	if debug {
+		log.Println(reqID, req.Method, req.URL)
+	}
+
+	remoteAddr := &net.TCPAddr{
+		IP:   nil,
+		Port: -1,
+	}
+
+	if s.useHTTP {
+		remoteAddr.IP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
+		if parsedPort, err := strconv.ParseInt(req.Header.Get("X-Client-Port"), 10, 0); err == nil {
+			remoteAddr.Port = int(parsedPort)
+		}
+	} else {
+		var err error
+		remoteAddr, err = net.ResolveTCPAddr("tcp", req.RemoteAddr)
+		if err != nil {
+			log.Println("remoteAddr:", err)
+			lw.Header().Set("Retry-After", errorRetryAfterString())
+			http.Error(lw, "Internal Server Error", http.StatusInternalServerError)
+			apiRequestsTotal.WithLabelValues("no_remote_addr").Inc()
+			return
+		}
+	}
+
+	switch req.Method {
+	case "GET":
+		s.handleGET(ctx, lw, req)
+	case "POST":
+		s.handlePOST(ctx, remoteAddr, lw, req)
+	default:
+		http.Error(lw, "Method Not Allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *apiSrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
+	reqID := ctx.Value(idKey).(requestID)
+
+	deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
+	if err != nil {
+		if debug {
+			log.Println(reqID, "bad device param")
+		}
+		lookupRequestsTotal.WithLabelValues("bad_request").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	key := deviceID.String()
+	rec, err := s.db.get(key)
+	if err != nil {
+		// some sort of internal error
+		lookupRequestsTotal.WithLabelValues("internal_error").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+
+	if len(rec.Addresses) == 0 {
+		lookupRequestsTotal.WithLabelValues("not_found").Inc()
+
+		s.mapsMut.Lock()
+		misses := s.misses[key]
+		if misses < rec.Misses {
+			misses = rec.Misses + 1
+		} else {
+			misses++
+		}
+		s.misses[key] = misses
+		s.mapsMut.Unlock()
+
+		if misses%notFoundMissesWriteInterval == 0 {
+			rec.Misses = misses
+			rec.Missed = time.Now().UnixNano()
+			rec.Addresses = nil
+			// rec.Seen retained from get
+			s.db.put(key, rec)
+		}
+
+		w.Header().Set("Retry-After", notFoundRetryAfterString(int(misses)))
+		http.Error(w, "Not Found", http.StatusNotFound)
+		return
+	}
+
+	lookupRequestsTotal.WithLabelValues("success").Inc()
+
+	bs, _ := json.Marshal(announcement{
+		Seen:      time.Unix(0, rec.Seen),
+		Addresses: addressStrs(rec.Addresses),
+	})
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bs)
+}
+
+func (s *apiSrv) handlePOST(ctx context.Context, remoteAddr *net.TCPAddr, w http.ResponseWriter, req *http.Request) {
+	reqID := ctx.Value(idKey).(requestID)
+
+	rawCert, err := certificateBytes(req)
+	if err != nil {
+		if debug {
+			log.Println(reqID, "no certificates:", err)
+		}
+		announceRequestsTotal.WithLabelValues("no_certificate").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Forbidden", http.StatusForbidden)
+		return
+	}
+
+	var ann announcement
+	if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
+		if debug {
+			log.Println(reqID, "decode:", err)
+		}
+		announceRequestsTotal.WithLabelValues("bad_request").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	deviceID := protocol.NewDeviceID(rawCert)
+
+	addresses := fixupAddresses(remoteAddr, ann.Addresses)
+	if len(addresses) == 0 {
+		announceRequestsTotal.WithLabelValues("bad_request").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	if err := s.handleAnnounce(deviceID, addresses); err != nil {
+		announceRequestsTotal.WithLabelValues("internal_error").Inc()
+		w.Header().Set("Retry-After", errorRetryAfterString())
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+
+	announceRequestsTotal.WithLabelValues("success").Inc()
+
+	w.Header().Set("Reannounce-After", reannounceAfterString())
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func (s *apiSrv) Stop() {
+	s.listener.Close()
+}
+
+func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string) error {
+	key := deviceID.String()
+	now := time.Now()
+	expire := now.Add(addressExpiryTime).UnixNano()
+
+	dbAddrs := make([]DatabaseAddress, len(addresses))
+	for i := range addresses {
+		dbAddrs[i].Address = addresses[i]
+		dbAddrs[i].Expires = expire
+	}
+
+	// The address slice must always be sorted for database merges to work
+	// properly.
+	sort.Sort(databaseAddressOrder(dbAddrs))
+
+	seen := now.UnixNano()
+	if s.repl != nil {
+		s.repl.send(key, dbAddrs, seen)
+	}
+	return s.db.merge(key, dbAddrs, seen)
+}
+
+func handlePing(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(204)
+}
+
+func certificateBytes(req *http.Request) ([]byte, error) {
+	if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
+		return req.TLS.PeerCertificates[0].Raw, nil
+	}
+
+	var bs []byte
+
+	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
+		if strings.Contains(hdr, "%") {
+			// Nginx using $ssl_client_escaped_cert
+			// The certificate is in PEM format with url encoding.
+			// We need to decode for the PEM decoder
+			hdr, err := url.QueryUnescape(hdr)
+			if err != nil {
+				// Decoding failed
+				return nil, err
+			}
+
+			bs = []byte(hdr)
+		} else {
+			// Nginx using $ssl_client_cert
+			// The certificate is in PEM format but with spaces for newlines. We
+			// need to reinstate the newlines for the PEM decoder. But we need to
+			// leave the spaces in the BEGIN and END lines - the first and last
+			// space - alone.
+			bs = []byte(hdr)
+			firstSpace := bytes.Index(bs, []byte(" "))
+			lastSpace := bytes.LastIndex(bs, []byte(" "))
+			for i := firstSpace + 1; i < lastSpace; i++ {
+				if bs[i] == ' ' {
+					bs[i] = '\n'
+				}
+			}
+		}
+	} else if hdr := req.Header.Get("X-Tls-Client-Cert-Der-Base64"); hdr != "" {
+		// Caddy {tls_client_certificate_der_base64}
+		hdr, err := base64.StdEncoding.DecodeString(hdr)
+		if err != nil {
+			// Decoding failed
+			return nil, err
+		}
+
+		bs = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: hdr})
+	} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
+		// Traefik 2 passtlsclientcert
+		// The certificate is in PEM format with url encoding but without newlines
+		// and start/end statements. We need to decode, reinstate the newlines every 64
+		// character and add statements for the PEM decoder
+		hdr, err := url.QueryUnescape(hdr)
+		if err != nil {
+			// Decoding failed
+			return nil, err
+		}
+
+		for i := 64; i < len(hdr); i += 65 {
+			hdr = hdr[:i] + "\n" + hdr[i:]
+		}
+
+		hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
+		hdr = hdr + "\n-----END CERTIFICATE-----\n"
+		bs = []byte(hdr)
+	}
+
+	if bs == nil {
+		return nil, errors.New("empty certificate header")
+	}
+
+	block, _ := pem.Decode(bs)
+	if block == nil {
+		// Decoding failed
+		return nil, errors.New("certificate decode result is empty")
+	}
+
+	return block.Bytes, nil
+}
+
+// fixupAddresses checks the list of addresses, removing invalid ones and
+// replacing unspecified IPs with the given remote IP.
+func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
+	fixed := make([]string, 0, len(addresses))
+	for _, annAddr := range addresses {
+		uri, err := url.Parse(annAddr)
+		if err != nil {
+			continue
+		}
+
+		host, port, err := net.SplitHostPort(uri.Host)
+		if err != nil {
+			continue
+		}
+
+		ip := net.ParseIP(host)
+
+		// Some classes of IP are no-go.
+		if ip.IsLoopback() || ip.IsMulticast() {
+			continue
+		}
+
+		if remote != nil {
+			if host == "" || ip.IsUnspecified() {
+				// Replace the unspecified IP with the request source.
+
+				// ... unless the request source is the loopback address or
+				// multicast/unspecified (can't happen, really).
+				if remote.IP.IsLoopback() || remote.IP.IsMulticast() || remote.IP.IsUnspecified() {
+					continue
+				}
+
+				// Do not use IPv6 remote address if requested scheme is ...4
+				// (i.e., tcp4, etc.)
+				if strings.HasSuffix(uri.Scheme, "4") && remote.IP.To4() == nil {
+					continue
+				}
+
+				// Do not use IPv4 remote address if requested scheme is ...6
+				if strings.HasSuffix(uri.Scheme, "6") && remote.IP.To4() != nil {
+					continue
+				}
+
+				host = remote.IP.String()
+			}
+
+			// If zero port was specified, use remote port.
+			if port == "0" && remote.Port > 0 {
+				port = strconv.Itoa(remote.Port)
+			}
+		}
+
+		uri.Host = net.JoinHostPort(host, port)
+		fixed = append(fixed, uri.String())
+	}
+
+	return fixed
+}
+
+type loggingResponseWriter struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
+	return &loggingResponseWriter{w, http.StatusOK}
+}
+
+func (lrw *loggingResponseWriter) WriteHeader(code int) {
+	lrw.statusCode = code
+	lrw.ResponseWriter.WriteHeader(code)
+}
+
+func addressStrs(dbAddrs []DatabaseAddress) []string {
+	res := make([]string, len(dbAddrs))
+	for i, a := range dbAddrs {
+		res[i] = a.Address
+	}
+	return res
+}
+
+func errorRetryAfterString() string {
+	return strconv.Itoa(errorRetryAfterSeconds + rand.Intn(errorRetryFuzzSeconds))
+}
+
+func notFoundRetryAfterString(misses int) string {
+	retryAfterS := notFoundRetryMinSeconds + notFoundRetryIncSeconds*misses
+	if retryAfterS > notFoundRetryMaxSeconds {
+		retryAfterS = notFoundRetryMaxSeconds
+	}
+	retryAfterS += rand.Intn(notFoundRetryFuzzSeconds)
+	return strconv.Itoa(retryAfterS)
+}
+
+func reannounceAfterString() string {
+	return strconv.Itoa(reannounceAfterSeconds + rand.Intn(reannounzeFuzzSeconds))
+}

cmd/stdiscosrv/apisrv_test.go

@@ -0,0 +1,88 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"net"
+	"testing"
+)
+
+func TestFixupAddresses(t *testing.T) {
+	cases := []struct {
+		remote *net.TCPAddr
+		in     []string
+		out    []string
+	}{
+		{ // verbatim passthrough
+			in:  []string{"tcp://1.2.3.4:22000"},
+			out: []string{"tcp://1.2.3.4:22000"},
+		}, { // unspecified replaced by remote
+			remote: addr("1.2.3.4", 22000),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://1.2.3.4:22000", "tcp://192.0.2.42:22000"},
+		}, { // unspecified not used as replacement
+			remote: addr("0.0.0.0", 22000),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // unspecified not used as replacement
+			remote: addr("::", 22000),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // localhost not used as replacement
+			remote: addr("127.0.0.1", 22000),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // localhost not used as replacement
+			remote: addr("::1", 22000),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // multicast not used as replacement
+			remote: addr("224.0.0.1", 22000),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // multicast not used as replacement
+			remote: addr("ff80::42", 22000),
+			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // explicitly announced weirdness is also filtered
+			remote: addr("192.0.2.42", 22000),
+			in:     []string{"tcp://:22000", "tcp://127.1.2.3:22000", "tcp://[::1]:22000", "tcp://[ff80::42]:22000"},
+			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // port remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://0.0.0.0:0"},
+			out:    []string{"tcp://123.123.123.123:9000"},
+		}, { // unspecified port remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://:0"},
+			out:    []string{"tcp://123.123.123.123:9000"},
+		}, { // empty remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://"},
+			out:    []string{},
+		}, { // port only remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://44.44.44.44:0"},
+			out:    []string{"tcp://44.44.44.44:9000"},
+		},
+	}
+
+	for _, tc := range cases {
+		out := fixupAddresses(tc.remote, tc.in)
+		if fmt.Sprint(out) != fmt.Sprint(tc.out) {
+			t.Errorf("fixupAddresses(%v, %v) => %v, expected %v", tc.remote, tc.in, out, tc.out)
+		}
+	}
+}
+
+func addr(host string, port int) *net.TCPAddr {
+	return &net.TCPAddr{
+		IP:   net.ParseIP(host),
+		Port: port,
+	}
+}

cmd/stdiscosrv/clean.go

@@ -1,75 +0,0 @@
-// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-
-package main
-
-import (
-	"database/sql"
-	"log"
-	"time"
-)
-
-type cleansrv struct {
-	intv time.Duration
-	db   *sql.DB
-	prep map[string]*sql.Stmt
-}
-
-func (s *cleansrv) Serve() {
-	for {
-		time.Sleep(next(s.intv))
-
-		err := s.cleanOldEntries()
-		if err != nil {
-			log.Println("Clean:", err)
-		}
-	}
-}
-
-func (s *cleansrv) Stop() {
-	panic("stop unimplemented")
-}
-
-func (s *cleansrv) cleanOldEntries() (err error) {
-	var tx *sql.Tx
-	tx, err = s.db.Begin()
-	if err != nil {
-		return err
-	}
-
-	defer func() {
-		if err == nil {
-			err = tx.Commit()
-		} else {
-			tx.Rollback()
-		}
-	}()
-
-	res, err := tx.Stmt(s.prep["cleanAddress"]).Exec()
-	if err != nil {
-		return err
-	}
-	if rows, _ := res.RowsAffected(); rows > 0 {
-		log.Printf("Clean: %d old addresses", rows)
-	}
-
-	res, err = tx.Stmt(s.prep["cleanDevice"]).Exec()
-	if err != nil {
-		return err
-	}
-	if rows, _ := res.RowsAffected(); rows > 0 {
-		log.Printf("Clean: %d old devices", rows)
-	}
-
-	var devs, addrs int
-	row := tx.Stmt(s.prep["countDevice"]).QueryRow()
-	if err = row.Scan(&devs); err != nil {
-		return err
-	}
-	row = tx.Stmt(s.prep["countAddress"]).QueryRow()
-	if err = row.Scan(&addrs); err != nil {
-		return err
-	}
-
-	log.Printf("Database: %d devices, %d addresses", devs, addrs)
-	return nil
-}

cmd/stdiscosrv/database.go

@@ -0,0 +1,376 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//go:generate go run ../../proto/scripts/protofmt.go database.proto
+//go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
+
+package main
+
+import (
+	"context"
+	"log"
+	"sort"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+type clock interface {
+	Now() time.Time
+}
+
+type defaultClock struct{}
+
+func (defaultClock) Now() time.Time {
+	return time.Now()
+}
+
+type database interface {
+	put(key string, rec DatabaseRecord) error
+	merge(key string, addrs []DatabaseAddress, seen int64) error
+	get(key string) (DatabaseRecord, error)
+}
+
+type levelDBStore struct {
+	db         *leveldb.DB
+	inbox      chan func()
+	clock      clock
+	marshalBuf []byte
+}
+
+func newLevelDBStore(dir string) (*levelDBStore, error) {
+	db, err := leveldb.OpenFile(dir, levelDBOptions)
+	if err != nil {
+		return nil, err
+	}
+	return &levelDBStore{
+		db:    db,
+		inbox: make(chan func(), 16),
+		clock: defaultClock{},
+	}, nil
+}
+
+func (s *levelDBStore) put(key string, rec DatabaseRecord) error {
+	t0 := time.Now()
+	defer func() {
+		databaseOperationSeconds.WithLabelValues(dbOpPut).Observe(time.Since(t0).Seconds())
+	}()
+
+	rc := make(chan error)
+
+	s.inbox <- func() {
+		size := rec.Size()
+		if len(s.marshalBuf) < size {
+			s.marshalBuf = make([]byte, size)
+		}
+		n, _ := rec.MarshalTo(s.marshalBuf)
+		rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
+	}
+
+	err := <-rc
+	if err != nil {
+		databaseOperations.WithLabelValues(dbOpPut, dbResError).Inc()
+	} else {
+		databaseOperations.WithLabelValues(dbOpPut, dbResSuccess).Inc()
+	}
+
+	return err
+}
+
+func (s *levelDBStore) merge(key string, addrs []DatabaseAddress, seen int64) error {
+	t0 := time.Now()
+	defer func() {
+		databaseOperationSeconds.WithLabelValues(dbOpMerge).Observe(time.Since(t0).Seconds())
+	}()
+
+	rc := make(chan error)
+	newRec := DatabaseRecord{
+		Addresses: addrs,
+		Seen:      seen,
+	}
+
+	s.inbox <- func() {
+		// grab the existing record
+		oldRec, err := s.get(key)
+		if err != nil {
+			// "not found" is not an error from get, so this is serious
+			// stuff only
+			rc <- err
+			return
+		}
+		newRec = merge(newRec, oldRec)
+
+		// We replicate s.put() functionality here ourselves instead of
+		// calling it because we want to serialize our get above together
+		// with the put in the same function.
+		size := newRec.Size()
+		if len(s.marshalBuf) < size {
+			s.marshalBuf = make([]byte, size)
+		}
+		n, _ := newRec.MarshalTo(s.marshalBuf)
+		rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
+	}
+
+	err := <-rc
+	if err != nil {
+		databaseOperations.WithLabelValues(dbOpMerge, dbResError).Inc()
+	} else {
+		databaseOperations.WithLabelValues(dbOpMerge, dbResSuccess).Inc()
+	}
+
+	return err
+}
+
+func (s *levelDBStore) get(key string) (DatabaseRecord, error) {
+	t0 := time.Now()
+	defer func() {
+		databaseOperationSeconds.WithLabelValues(dbOpGet).Observe(time.Since(t0).Seconds())
+	}()
+
+	keyBs := []byte(key)
+	val, err := s.db.Get(keyBs, nil)
+	if err == leveldb.ErrNotFound {
+		databaseOperations.WithLabelValues(dbOpGet, dbResNotFound).Inc()
+		return DatabaseRecord{}, nil
+	}
+	if err != nil {
+		databaseOperations.WithLabelValues(dbOpGet, dbResError).Inc()
+		return DatabaseRecord{}, err
+	}
+
+	var rec DatabaseRecord
+
+	if err := rec.Unmarshal(val); err != nil {
+		databaseOperations.WithLabelValues(dbOpGet, dbResUnmarshalError).Inc()
+		return DatabaseRecord{}, nil
+	}
+
+	rec.Addresses = expire(rec.Addresses, s.clock.Now().UnixNano())
+	databaseOperations.WithLabelValues(dbOpGet, dbResSuccess).Inc()
+	return rec, nil
+}
+
+func (s *levelDBStore) Serve(ctx context.Context) error {
+	t := time.NewTimer(0)
+	defer t.Stop()
+	defer s.db.Close()
+
+	// Start the statistics serve routine. It will exit with us when
+	// statisticsTrigger is closed.
+	statisticsTrigger := make(chan struct{})
+	statisticsDone := make(chan struct{})
+	go s.statisticsServe(statisticsTrigger, statisticsDone)
+
+loop:
+	for {
+		select {
+		case fn := <-s.inbox:
+			// Run function in serialized order.
+			fn()
+
+		case <-t.C:
+			// Trigger the statistics routine to do its thing in the
+			// background.
+			statisticsTrigger <- struct{}{}
+
+		case <-statisticsDone:
+			// The statistics routine is done with one iteratation, schedule
+			// the next.
+			t.Reset(databaseStatisticsInterval)
+
+		case <-ctx.Done():
+			// We're done.
+			close(statisticsTrigger)
+			break loop
+		}
+	}
+
+	// Also wait for statisticsServe to return
+	<-statisticsDone
+
+	return nil
+}
+
+func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
+	defer close(done)
+
+	for range trigger {
+		t0 := time.Now()
+		nowNanos := t0.UnixNano()
+		cutoff24h := t0.Add(-24 * time.Hour).UnixNano()
+		cutoff1w := t0.Add(-7 * 24 * time.Hour).UnixNano()
+		cutoff2Mon := t0.Add(-60 * 24 * time.Hour).UnixNano()
+		current, last24h, last1w, inactive, errors := 0, 0, 0, 0, 0
+
+		iter := s.db.NewIterator(&util.Range{}, nil)
+		for iter.Next() {
+			// Attempt to unmarshal the record and count the
+			// failure if there's something wrong with it.
+			var rec DatabaseRecord
+			if err := rec.Unmarshal(iter.Value()); err != nil {
+				errors++
+				continue
+			}
+
+			// If there are addresses that have not expired it's a current
+			// record, otherwise account it based on when it was last seen
+			// (last 24 hours or last week) or finally as inactice.
+			switch {
+			case len(expire(rec.Addresses, nowNanos)) > 0:
+				current++
+			case rec.Seen > cutoff24h:
+				last24h++
+			case rec.Seen > cutoff1w:
+				last1w++
+			case rec.Seen > cutoff2Mon:
+				inactive++
+			case rec.Missed < cutoff2Mon:
+				// It hasn't been seen lately and we haven't recorded
+				// someone asking for this device in a long time either;
+				// delete the record.
+				if err := s.db.Delete(iter.Key(), nil); err != nil {
+					databaseOperations.WithLabelValues(dbOpDelete, dbResError).Inc()
+				} else {
+					databaseOperations.WithLabelValues(dbOpDelete, dbResSuccess).Inc()
+				}
+			default:
+				inactive++
+			}
+		}
+
+		iter.Release()
+
+		databaseKeys.WithLabelValues("current").Set(float64(current))
+		databaseKeys.WithLabelValues("last24h").Set(float64(last24h))
+		databaseKeys.WithLabelValues("last1w").Set(float64(last1w))
+		databaseKeys.WithLabelValues("inactive").Set(float64(inactive))
+		databaseKeys.WithLabelValues("error").Set(float64(errors))
+		databaseStatisticsSeconds.Set(time.Since(t0).Seconds())
+
+		// Signal that we are done and can be scheduled again.
+		done <- struct{}{}
+	}
+}
+
+// merge returns the merged result of the two database records a and b. The
+// result is the union of the two address sets, with the newer expiry time
+// chosen for any duplicates.
+func merge(a, b DatabaseRecord) DatabaseRecord {
+	// Both lists must be sorted for this to work.
+	if !sort.IsSorted(databaseAddressOrder(a.Addresses)) {
+		log.Println("Warning: bug: addresses not correctly sorted in merge")
+		a.Addresses = sortedAddressCopy(a.Addresses)
+	}
+	if !sort.IsSorted(databaseAddressOrder(b.Addresses)) {
+		// no warning because this is the side we read from disk and it may
+		// legitimately predate correct sorting.
+		b.Addresses = sortedAddressCopy(b.Addresses)
+	}
+
+	res := DatabaseRecord{
+		Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
+		Seen:      a.Seen,
+	}
+	if b.Seen > a.Seen {
+		res.Seen = b.Seen
+	}
+
+	aIdx := 0
+	bIdx := 0
+	aAddrs := a.Addresses
+	bAddrs := b.Addresses
+loop:
+	for {
+		switch {
+		case aIdx == len(aAddrs) && bIdx == len(bAddrs):
+			// both lists are exhausted, we are done
+			break loop
+
+		case aIdx == len(aAddrs):
+			// a is exhausted, pick from b and continue
+			res.Addresses = append(res.Addresses, bAddrs[bIdx])
+			bIdx++
+			continue
+
+		case bIdx == len(bAddrs):
+			// b is exhausted, pick from a and continue
+			res.Addresses = append(res.Addresses, aAddrs[aIdx])
+			aIdx++
+			continue
+		}
+
+		// We have values left on both sides.
+		aVal := aAddrs[aIdx]
+		bVal := bAddrs[bIdx]
+
+		switch {
+		case aVal.Address == bVal.Address:
+			// update for same address, pick newer
+			if aVal.Expires > bVal.Expires {
+				res.Addresses = append(res.Addresses, aVal)
+			} else {
+				res.Addresses = append(res.Addresses, bVal)
+			}
+			aIdx++
+			bIdx++
+
+		case aVal.Address < bVal.Address:
+			// a is smallest, pick it and continue
+			res.Addresses = append(res.Addresses, aVal)
+			aIdx++
+
+		default:
+			// b is smallest, pick it and continue
+			res.Addresses = append(res.Addresses, bVal)
+			bIdx++
+		}
+	}
+	return res
+}
+
+// expire returns the list of addresses after removing expired entries.
+// Expiration happen in place, so the slice given as the parameter is
+// destroyed. Internal order is not preserved.
+func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
+	i := 0
+	for i < len(addrs) {
+		if addrs[i].Expires < now {
+			// This item is expired. Replace it with the last in the list
+			// (noop if we are at the last item).
+			addrs[i] = addrs[len(addrs)-1]
+			// Wipe the last item of the list to release references to
+			// strings and stuff.
+			addrs[len(addrs)-1] = DatabaseAddress{}
+			// Shorten the slice.
+			addrs = addrs[:len(addrs)-1]
+			continue
+		}
+		i++
+	}
+	return addrs
+}
+
+func sortedAddressCopy(addrs []DatabaseAddress) []DatabaseAddress {
+	sorted := make([]DatabaseAddress, len(addrs))
+	copy(sorted, addrs)
+	sort.Sort(databaseAddressOrder(sorted))
+	return sorted
+}
+
+type databaseAddressOrder []DatabaseAddress
+
+func (s databaseAddressOrder) Less(a, b int) bool {
+	return s[a].Address < s[b].Address
+}
+
+func (s databaseAddressOrder) Swap(a, b int) {
+	s[a], s[b] = s[b], s[a]
+}
+
+func (s databaseAddressOrder) Len() int {
+	return len(s)
+}

cmd/stdiscosrv/database.pb.go

@@ -0,0 +1,847 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: database.proto
+
+package main
+
+import (
+	fmt "fmt"
+	_ "github.com/gogo/protobuf/gogoproto"
+	proto "github.com/gogo/protobuf/proto"
+	io "io"
+	math "math"
+	math_bits "math/bits"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+
+type DatabaseRecord struct {
+	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
+	Misses    int32             `protobuf:"varint,2,opt,name=misses,proto3" json:"misses,omitempty"`
+	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
+	Missed    int64             `protobuf:"varint,4,opt,name=missed,proto3" json:"missed,omitempty"`
+}
+
+func (m *DatabaseRecord) Reset()         { *m = DatabaseRecord{} }
+func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
+func (*DatabaseRecord) ProtoMessage()    {}
+func (*DatabaseRecord) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{0}
+}
+func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseRecord.Merge(m, src)
+}
+func (m *DatabaseRecord) XXX_Size() int {
+	return m.Size()
+}
+func (m *DatabaseRecord) XXX_DiscardUnknown() {
+	xxx_messageInfo_DatabaseRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DatabaseRecord proto.InternalMessageInfo
+
+type ReplicationRecord struct {
+	Key       string            `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+	Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses"`
+	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
+}
+
+func (m *ReplicationRecord) Reset()         { *m = ReplicationRecord{} }
+func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
+func (*ReplicationRecord) ProtoMessage()    {}
+func (*ReplicationRecord) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{1}
+}
+func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ReplicationRecord.Merge(m, src)
+}
+func (m *ReplicationRecord) XXX_Size() int {
+	return m.Size()
+}
+func (m *ReplicationRecord) XXX_DiscardUnknown() {
+	xxx_messageInfo_ReplicationRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ReplicationRecord proto.InternalMessageInfo
+
+type DatabaseAddress struct {
+	Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
+	Expires int64  `protobuf:"varint,2,opt,name=expires,proto3" json:"expires,omitempty"`
+}
+
+func (m *DatabaseAddress) Reset()         { *m = DatabaseAddress{} }
+func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
+func (*DatabaseAddress) ProtoMessage()    {}
+func (*DatabaseAddress) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{2}
+}
+func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseAddress.Merge(m, src)
+}
+func (m *DatabaseAddress) XXX_Size() int {
+	return m.Size()
+}
+func (m *DatabaseAddress) XXX_DiscardUnknown() {
+	xxx_messageInfo_DatabaseAddress.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DatabaseAddress proto.InternalMessageInfo
+
+func init() {
+	proto.RegisterType((*DatabaseRecord)(nil), "main.DatabaseRecord")
+	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
+	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
+}
+
+func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
+
+var fileDescriptor_b90fe3356ea5df07 = []byte{
+	// 270 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
+	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
+	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
+	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
+	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
+	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
+	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
+	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
+	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
+	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
+	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
+	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
+	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
+	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
+	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
+	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
+	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
+}
+
+func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Missed != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+		i--
+		dAtA[i] = 0x20
+	}
+	if m.Seen != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
+	}
+	if m.Misses != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+		i--
+		dAtA[i] = 0x10
+	}
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Seen != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
+	}
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x12
+		}
+	}
+	if len(m.Key) > 0 {
+		i -= len(m.Key)
+		copy(dAtA[i:], m.Key)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Expires != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
+		i--
+		dAtA[i] = 0x10
+	}
+	if len(m.Address) > 0 {
+		i -= len(m.Address)
+		copy(dAtA[i:], m.Address)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
+	offset -= sovDatabase(v)
+	base := offset
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return base
+}
+func (m *DatabaseRecord) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Addresses) > 0 {
+		for _, e := range m.Addresses {
+			l = e.Size()
+			n += 1 + l + sovDatabase(uint64(l))
+		}
+	}
+	if m.Misses != 0 {
+		n += 1 + sovDatabase(uint64(m.Misses))
+	}
+	if m.Seen != 0 {
+		n += 1 + sovDatabase(uint64(m.Seen))
+	}
+	if m.Missed != 0 {
+		n += 1 + sovDatabase(uint64(m.Missed))
+	}
+	return n
+}
+
+func (m *ReplicationRecord) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Key)
+	if l > 0 {
+		n += 1 + l + sovDatabase(uint64(l))
+	}
+	if len(m.Addresses) > 0 {
+		for _, e := range m.Addresses {
+			l = e.Size()
+			n += 1 + l + sovDatabase(uint64(l))
+		}
+	}
+	if m.Seen != 0 {
+		n += 1 + sovDatabase(uint64(m.Seen))
+	}
+	return n
+}
+
+func (m *DatabaseAddress) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Address)
+	if l > 0 {
+		n += 1 + l + sovDatabase(uint64(l))
+	}
+	if m.Expires != 0 {
+		n += 1 + sovDatabase(uint64(m.Expires))
+	}
+	return n
+}
+
+func sovDatabase(x uint64) (n int) {
+	return (math_bits.Len64(x|1) + 6) / 7
+}
+func sozDatabase(x uint64) (n int) {
+	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: DatabaseRecord: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: DatabaseRecord: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Addresses = append(m.Addresses, DatabaseAddress{})
+			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Misses", wireType)
+			}
+			m.Misses = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Misses |= int32(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
+			}
+			m.Seen = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Seen |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Missed", wireType)
+			}
+			m.Missed = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Missed |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ReplicationRecord: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ReplicationRecord: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Key = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Addresses = append(m.Addresses, DatabaseAddress{})
+			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
+			}
+			m.Seen = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Seen |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: DatabaseAddress: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: DatabaseAddress: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Address", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Address = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Expires", wireType)
+			}
+			m.Expires = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Expires |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipDatabase(dAtA []byte) (n int, err error) {
+	l := len(dAtA)
+	iNdEx := 0
+	depth := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if dAtA[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+		case 1:
+			iNdEx += 8
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if length < 0 {
+				return 0, ErrInvalidLengthDatabase
+			}
+			iNdEx += length
+		case 3:
+			depth++
+		case 4:
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupDatabase
+			}
+			depth--
+		case 5:
+			iNdEx += 4
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthDatabase
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
+	}
+	return 0, io.ErrUnexpectedEOF
+}
+
+var (
+	ErrInvalidLengthDatabase        = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowDatabase          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupDatabase = fmt.Errorf("proto: unexpected end of group")
+)

cmd/stdiscosrv/database.proto

@@ -0,0 +1,36 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+syntax = "proto3";
+
+package main;
+
+import "repos/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.goproto_unkeyed_all) = false;
+option (gogoproto.goproto_unrecognized_all) = false;
+option (gogoproto.goproto_sizecache_all) = false;
+
+message DatabaseRecord {
+    repeated DatabaseAddress addresses = 1 [(gogoproto.nullable) = false];
+    int32                    misses    = 2; // Number of lookups* without hits
+    int64                    seen      = 3; // Unix nanos, last device announce
+    int64                    missed    = 4; // Unix nanos, last* failed lookup
+}
+
+// *) Not every lookup results in a write, so may not be completely accurate
+
+message ReplicationRecord {
+    string                   key       = 1;
+    repeated DatabaseAddress addresses = 2 [(gogoproto.nullable) = false];
+    int64                    seen      = 3; // Unix nanos, last device announce
+}
+
+message DatabaseAddress {
+    string address = 1;
+    int64  expires = 2; // Unix nanos
+}

cmd/stdiscosrv/database_test.go

@@ -0,0 +1,213 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestDatabaseGetSet(t *testing.T) {
+	os.RemoveAll("_database")
+	defer os.RemoveAll("_database")
+	db, err := newLevelDBStore("_database")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	go db.Serve(ctx)
+	defer cancel()
+
+	// Check missing record
+
+	rec, err := db.get("abcd")
+	if err != nil {
+		t.Error("not found should not be an error")
+	}
+	if len(rec.Addresses) != 0 {
+		t.Error("addresses should be empty")
+	}
+	if rec.Misses != 0 {
+		t.Error("missing should be zero")
+	}
+
+	// Set up a clock
+
+	now := time.Now()
+	tc := &testClock{now}
+	db.clock = tc
+
+	// Put a record
+
+	rec.Addresses = []DatabaseAddress{
+		{Address: "tcp://1.2.3.4:5", Expires: tc.Now().Add(time.Minute).UnixNano()},
+	}
+	if err := db.put("abcd", rec); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it
+
+	rec, err = db.get("abcd")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 1 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have one address")
+	}
+	if rec.Addresses[0].Address != "tcp://1.2.3.4:5" {
+		t.Log(rec.Addresses)
+		t.Error("incorrect address")
+	}
+
+	// Wind the clock one half expiry, and merge in a new address
+
+	tc.wind(30 * time.Second)
+
+	addrs := []DatabaseAddress{
+		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
+	}
+	if err := db.merge("abcd", addrs, tc.Now().UnixNano()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it
+
+	rec, err = db.get("abcd")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 2 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have two addresses")
+	}
+	if rec.Addresses[0].Address != "tcp://1.2.3.4:5" {
+		t.Log(rec.Addresses)
+		t.Error("incorrect address[0]")
+	}
+	if rec.Addresses[1].Address != "tcp://6.7.8.9:0" {
+		t.Log(rec.Addresses)
+		t.Error("incorrect address[1]")
+	}
+
+	// Pass the first expiry time
+
+	tc.wind(45 * time.Second)
+
+	// Verify it
+
+	rec, err = db.get("abcd")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 1 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have one address")
+	}
+	if rec.Addresses[0].Address != "tcp://6.7.8.9:0" {
+		t.Log(rec.Addresses)
+		t.Error("incorrect address")
+	}
+
+	// Put a record with misses
+
+	rec = DatabaseRecord{Misses: 42}
+	if err := db.put("efgh", rec); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it
+
+	rec, err = db.get("efgh")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 0 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have no addresses")
+	}
+	if rec.Misses != 42 {
+		t.Log(rec.Misses)
+		t.Error("incorrect misses")
+	}
+
+	// Set an address
+
+	addrs = []DatabaseAddress{
+		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
+	}
+	if err := db.merge("efgh", addrs, tc.Now().UnixNano()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it
+
+	rec, err = db.get("efgh")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 1 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have one address")
+	}
+	if rec.Misses != 0 {
+		t.Log(rec.Misses)
+		t.Error("should have no misses")
+	}
+}
+
+func TestFilter(t *testing.T) {
+	// all cases are expired with t=10
+	cases := []struct {
+		a []DatabaseAddress
+		b []DatabaseAddress
+	}{
+		{
+			a: nil,
+			b: nil,
+		},
+		{
+			a: []DatabaseAddress{{Address: "a", Expires: 9}, {Address: "b", Expires: 9}, {Address: "c", Expires: 9}},
+			b: []DatabaseAddress{},
+		},
+		{
+			a: []DatabaseAddress{{Address: "a", Expires: 10}},
+			b: []DatabaseAddress{{Address: "a", Expires: 10}},
+		},
+		{
+			a: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
+			b: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
+		},
+		{
+			a: []DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
+			b: []DatabaseAddress{{Address: "d", Expires: 15}, {Address: "b", Expires: 15}}, // gets reordered
+		},
+	}
+
+	for _, tc := range cases {
+		res := expire(tc.a, 10)
+		if fmt.Sprint(res) != fmt.Sprint(tc.b) {
+			t.Errorf("Incorrect result %v, expected %v", res, tc.b)
+		}
+	}
+}
+
+type testClock struct {
+	now time.Time
+}
+
+func (t *testClock) wind(d time.Duration) {
+	t.now = t.now.Add(d)
+}
+
+func (t *testClock) Now() time.Time {
+	return t.now
+}

cmd/stdiscosrv/db.go

@@ -1,32 +0,0 @@
-// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-
-package main
-
-import (
-	"database/sql"
-	"fmt"
-)
-
-type setupFunc func(db *sql.DB) error
-type compileFunc func(db *sql.DB) (map[string]*sql.Stmt, error)
-
-var (
-	setupFuncs   = make(map[string]setupFunc)
-	compileFuncs = make(map[string]compileFunc)
-)
-
-func register(name string, setup setupFunc, compile compileFunc) {
-	setupFuncs[name] = setup
-	compileFuncs[name] = compile
-}
-
-func setup(backend string, db *sql.DB) (map[string]*sql.Stmt, error) {
-	setup, ok := setupFuncs[backend]
-	if !ok {
-		return nil, fmt.Errorf("Unsupported backend")
-	}
-	if err := setup(db); err != nil {
-		return nil, err
-	}
-	return compileFuncs[backend](db)
-}

cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv

@@ -0,0 +1,4 @@
+[stdiscosrv]
+title=Syncthing discovery server
+description=Lets syncthing clients discover each other
+ports=8443/tcp

cmd/stdiscosrv/etc/linux-systemd/default

@@ -0,0 +1,3 @@
+# Default settings for syncthing-discosrv (stdiscosrv).
+## Add Options here:
+DISCOSRV_OPTS=

cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=Syncthing Discovery Server
+After=network.target
+Documentation=man:stdiscosrv(1)
+
+[Service]
+WorkingDirectory=/var/lib/syncthing-discosrv
+EnvironmentFile=/etc/default/syncthing-discosrv
+ExecStart=/usr/bin/stdiscosrv $DISCOSRV_OPTS
+
+# Hardening
+User=syncthing-discosrv
+Group=syncthing
+ProtectSystem=strict
+ReadWritePaths=/var/lib/syncthing-discosrv
+NoNewPrivileges=true
+PrivateTmp=true
+PrivateDevices=true
+ProtectHome=true
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=true
+
+[Install]
+WantedBy=multi-user.target
+Alias=syncthing-discosrv.service

cmd/stdiscosrv/main.go

@@ -1,146 +1,194 @@
-// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
+	"context"
 	"crypto/tls"
-	"database/sql"
 	"flag"
-	"fmt"
 	"log"
+	"net"
+	"net/http"
 	"os"
-	"runtime"
-	"strconv"
+	"strings"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
-	"github.com/thejerf/suture"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/thejerf/suture/v4"
 )
 
 const (
-	minNegCache  = 60        // seconds
-	maxNegCache  = 3600      // seconds
-	maxDeviceAge = 7 * 86400 // one week, in seconds
+	addressExpiryTime          = 2 * time.Hour
+	databaseStatisticsInterval = 5 * time.Minute
+
+	// Reannounce-After is set to reannounceAfterSeconds +
+	// random(reannounzeFuzzSeconds), similar for Retry-After
+	reannounceAfterSeconds = 3300
+	reannounzeFuzzSeconds  = 300
+	errorRetryAfterSeconds = 1500
+	errorRetryFuzzSeconds  = 300
+
+	// Retry for not found is minSeconds + failures * incSeconds +
+	// random(fuzz), where failures is the number of consecutive lookups
+	// with no answer, up to maxSeconds. The fuzz is applied after capping
+	// to maxSeconds.
+	notFoundRetryMinSeconds  = 60
+	notFoundRetryMaxSeconds  = 3540
+	notFoundRetryIncSeconds  = 10
+	notFoundRetryFuzzSeconds = 60
+
+	// How often (in requests) we serialize the missed counter to database.
+	notFoundMissesWriteInterval = 10
+
+	httpReadTimeout    = 5 * time.Second
+	httpWriteTimeout   = 5 * time.Second
+	httpMaxHeaderBytes = 1 << 10
+
+	// Size of the replication outbox channel
+	replicationOutboxSize = 10000
 )
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`stdiscosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
+// These options make the database a little more optimized for writes, at
+// the expense of some memory usage and risk of losing writes in a (system)
+// crash.
+var levelDBOptions = &opt.Options{
+	NoSync:      true,
+	WriteBuffer: 32 << 20, // default 4<<20
 }
 
 var (
-	lruSize     = 10240
-	limitAvg    = 5
-	limitBurst  = 20
-	globalStats stats
-	statsFile   string
-	backend     = "ql"
-	dsn         = getEnvDefault("STDISCOSRV_DB_DSN", "memory://stdiscosrv")
-	certFile    = "cert.pem"
-	keyFile     = "key.pem"
-	debug       = false
-	useHTTP     = false
+	debug = false
 )
 
 func main() {
-	const (
-		cleanIntv = 1 * time.Hour
-		statsIntv = 5 * time.Minute
-	)
-
 	var listen string
+	var dir string
+	var metricsListen string
+	var replicationListen string
+	var replicationPeers string
+	var certFile string
+	var keyFile string
+	var useHTTP bool
 
 	log.SetOutput(os.Stdout)
 	log.SetFlags(0)
 
+	flag.StringVar(&certFile, "cert", "./cert.pem", "Certificate file")
+	flag.StringVar(&dir, "db-dir", "./discovery.db", "Database directory")
+	flag.BoolVar(&debug, "debug", false, "Print debug output")
+	flag.BoolVar(&useHTTP, "http", false, "Listen on HTTP (behind an HTTPS proxy)")
 	flag.StringVar(&listen, "listen", ":8443", "Listen address")
-	flag.IntVar(&lruSize, "limit-cache", lruSize, "Limiter cache entries")
-	flag.IntVar(&limitAvg, "limit-avg", limitAvg, "Allowed average package rate, per 10 s")
-	flag.IntVar(&limitBurst, "limit-burst", limitBurst, "Allowed burst size, packets")
-	flag.StringVar(&statsFile, "stats-file", statsFile, "File to write periodic operation stats to")
-	flag.StringVar(&backend, "db-backend", backend, "Database backend to use")
-	flag.StringVar(&dsn, "db-dsn", dsn, "Database DSN")
-	flag.StringVar(&certFile, "cert", certFile, "Certificate file")
-	flag.StringVar(&keyFile, "key", keyFile, "Key file")
-	flag.BoolVar(&debug, "debug", debug, "Debug")
-	flag.BoolVar(&useHTTP, "http", useHTTP, "Listen on HTTP (behind an HTTPS proxy)")
+	flag.StringVar(&keyFile, "key", "./key.pem", "Key file")
+	flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
+	flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
+	flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
-	log.Println(LongVersion)
+	log.Println(build.LongVersionFor("stdiscosrv"))
+	if *showVersion {
+		return
+	}
 
-	var cert tls.Certificate
-	var err error
-	if !useHTTP {
-		cert, err = tls.LoadX509KeyPair(certFile, keyFile)
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if os.IsNotExist(err) {
+		log.Println("Failed to load keypair. Generating one, this might take a while...")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 20*365)
 		if err != nil {
-			log.Println("Failed to load keypair. Generating one, this might take a while...")
-			cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 3072)
-			if err != nil {
-				log.Fatalln("Failed to generate X509 key pair:", err)
-			}
+			log.Fatalln("Failed to generate X509 key pair:", err)
+		}
+	} else if err != nil {
+		log.Fatalln("Failed to load keypair:", err)
+	}
+	devID := protocol.NewDeviceID(cert.Certificate[0])
+	log.Println("Server device ID is", devID)
+
+	// Parse the replication specs, if any.
+	var allowedReplicationPeers []protocol.DeviceID
+	var replicationDestinations []string
+	parts := strings.Split(replicationPeers, ",")
+	for _, part := range parts {
+		if part == "" {
+			continue
 		}
 
-		devID := protocol.NewDeviceID(cert.Certificate[0])
-		log.Println("Server device ID is", devID)
+		fields := strings.Split(part, "@")
+		switch len(fields) {
+		case 2:
+			// This is an id@address specification. Grab the address for the
+			// destination list. Try to resolve it once to catch obvious
+			// syntax errors here rather than having the sender service fail
+			// repeatedly later.
+			_, err := net.ResolveTCPAddr("tcp", fields[1])
+			if err != nil {
+				log.Fatalln("Resolving address:", err)
+			}
+			replicationDestinations = append(replicationDestinations, fields[1])
+			fallthrough // N.B.
+
+		case 1:
+			// The first part is always a device ID.
+			id, err := protocol.DeviceIDFromString(fields[0])
+			if err != nil {
+				log.Fatalln("Parsing device ID:", err)
+			}
+			if id == protocol.EmptyDeviceID {
+				log.Fatalf("Missing device ID for peer in %q", part)
+			}
+			allowedReplicationPeers = append(allowedReplicationPeers, id)
+
+		default:
+			log.Fatalln("Unrecognized replication spec:", part)
+		}
 	}
 
-	db, err := sql.Open(backend, dsn)
+	// Root of the service tree.
+	main := suture.New("main", suture.Spec{
+		PassThroughPanics: true,
+	})
+
+	// Start the database.
+	db, err := newLevelDBStore(dir)
 	if err != nil {
-		log.Fatalln("sql.Open:", err)
+		log.Fatalln("Open database:", err)
 	}
-	prep, err := setup(backend, db)
-	if err != nil {
-		log.Fatalln("Setup:", err)
+	main.Add(db)
+
+	// Start any replication senders.
+	var repl replicationMultiplexer
+	for _, dst := range replicationDestinations {
+		rs := newReplicationSender(dst, cert, allowedReplicationPeers)
+		main.Add(rs)
+		repl = append(repl, rs)
 	}
 
-	main := suture.NewSimple("main")
-
-	main.Add(&querysrv{
-		addr: listen,
-		cert: cert,
-		db:   db,
-		prep: prep,
-	})
-
-	main.Add(&cleansrv{
-		intv: cleanIntv,
-		db:   db,
-		prep: prep,
-	})
-
-	main.Add(&statssrv{
-		intv: statsIntv,
-		file: statsFile,
-		db:   db,
-	})
-
-	globalStats.Reset()
-	main.Serve()
-}
-
-func getEnvDefault(key, def string) string {
-	if val := os.Getenv(key); val != "" {
-		return val
+	// If we have replication configured, start the replication listener.
+	if len(allowedReplicationPeers) > 0 {
+		rl := newReplicationListener(replicationListen, cert, allowedReplicationPeers, db)
+		main.Add(rl)
 	}
-	return def
-}
 
-func next(intv time.Duration) time.Duration {
-	t0 := time.Now()
-	t1 := t0.Add(intv).Truncate(intv)
-	return t1.Sub(t0)
+	// Start the main API server.
+	qs := newAPISrv(listen, cert, db, repl, useHTTP)
+	main.Add(qs)
+
+	// If we have a metrics port configured, start a metrics handler.
+	if metricsListen != "" {
+		go func() {
+			mux := http.NewServeMux()
+			mux.Handle("/metrics", promhttp.Handler())
+			log.Fatal(http.ListenAndServe(metricsListen, mux))
+		}()
+	}
+
+	// Engage!
+	main.Serve(context.Background())
 }

cmd/stdiscosrv/psql.go

@@ -1,98 +0,0 @@
-// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-
-package main
-
-import (
-	"database/sql"
-	"fmt"
-
-	_ "github.com/lib/pq"
-)
-
-func init() {
-	register("postgres", postgresSetup, postgresCompile)
-}
-
-func postgresSetup(db *sql.DB) error {
-	var err error
-
-	db.SetMaxIdleConns(4)
-	db.SetMaxOpenConns(8)
-
-	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Devices (
-		DeviceID CHAR(63) NOT NULL PRIMARY KEY,
-		Seen TIMESTAMP NOT NULL
-	)`)
-	if err != nil {
-		return err
-	}
-
-	var tmp string
-	row := db.QueryRow(`SELECT 'DevicesDeviceIDIndex'::regclass`)
-	if err = row.Scan(&tmp); err != nil {
-		_, err = db.Exec(`CREATE INDEX DevicesDeviceIDIndex ON Devices (DeviceID)`)
-	}
-	if err != nil {
-		return err
-	}
-
-	row = db.QueryRow(`SELECT 'DevicesSeenIndex'::regclass`)
-	if err = row.Scan(&tmp); err != nil {
-		_, err = db.Exec(`CREATE INDEX DevicesSeenIndex ON Devices (Seen)`)
-	}
-	if err != nil {
-		return err
-	}
-
-	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Addresses (
-		DeviceID CHAR(63) NOT NULL,
-		Seen TIMESTAMP NOT NULL,
-		Address VARCHAR(2048) NOT NULL
-	)`)
-	if err != nil {
-		return err
-	}
-
-	row = db.QueryRow(`SELECT 'AddressesDeviceIDSeenIndex'::regclass`)
-	if err = row.Scan(&tmp); err != nil {
-		_, err = db.Exec(`CREATE INDEX AddressesDeviceIDSeenIndex ON Addresses (DeviceID, Seen)`)
-	}
-	if err != nil {
-		return err
-	}
-
-	row = db.QueryRow(`SELECT 'AddressesDeviceIDAddressIndex'::regclass`)
-	if err = row.Scan(&tmp); err != nil {
-		_, err = db.Exec(`CREATE INDEX AddressesDeviceIDAddressIndex ON Addresses (DeviceID, Address)`)
-	}
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func postgresCompile(db *sql.DB) (map[string]*sql.Stmt, error) {
-	stmts := map[string]string{
-		"cleanAddress":  "DELETE FROM Addresses WHERE Seen < now() - '2 hour'::INTERVAL",
-		"cleanDevice":   fmt.Sprintf("DELETE FROM Devices WHERE Seen < now() - '%d hour'::INTERVAL", maxDeviceAge/3600),
-		"countAddress":  "SELECT count(*) FROM Addresses",
-		"countDevice":   "SELECT count(*) FROM Devices",
-		"insertAddress": "INSERT INTO Addresses (DeviceID, Seen, Address) VALUES ($1, now(), $2)",
-		"insertDevice":  "INSERT INTO Devices (DeviceID, Seen) VALUES ($1, now())",
-		"selectAddress": "SELECT Address FROM Addresses WHERE DeviceID=$1 AND Seen > now() - '1 hour'::INTERVAL ORDER BY random() LIMIT 16",
-		"selectDevice":  "SELECT Seen FROM Devices WHERE DeviceID=$1",
-		"updateAddress": "UPDATE Addresses SET Seen=now() WHERE DeviceID=$1 AND Address=$2",
-		"updateDevice":  "UPDATE Devices SET Seen=now() WHERE DeviceID=$1",
-	}
-
-	res := make(map[string]*sql.Stmt, len(stmts))
-	for key, stmt := range stmts {
-		prep, err := db.Prepare(stmt)
-		if err != nil {
-			return nil, err
-		}
-		res[key] = prep
-	}
-	return res, nil
-}

cmd/stdiscosrv/ql.go

@@ -1,81 +0,0 @@
-// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
-
-package main
-
-import (
-	"database/sql"
-	"fmt"
-	"log"
-
-	"github.com/cznic/ql"
-)
-
-func init() {
-	ql.RegisterDriver()
-	register("ql", qlSetup, qlCompile)
-}
-
-func qlSetup(db *sql.DB) (err error) {
-	tx, err := db.Begin()
-	if err != nil {
-		return
-	}
-
-	defer func() {
-		if err == nil {
-			err = tx.Commit()
-		} else {
-			tx.Rollback()
-		}
-	}()
-
-	_, err = tx.Exec(`CREATE TABLE IF NOT EXISTS Devices (
-		DeviceID STRING NOT NULL,
-		Seen TIME NOT NULL
-	)`)
-	if err != nil {
-		return
-	}
-
-	if _, err = tx.Exec(`CREATE INDEX IF NOT EXISTS DevicesDeviceIDIndex ON Devices (DeviceID)`); err != nil {
-		return
-	}
-
-	_, err = tx.Exec(`CREATE TABLE IF NOT EXISTS Addresses (
-		DeviceID STRING NOT NULL,
-		Seen TIME NOT NULL,
-		Address STRING NOT NULL,
-	)`)
-	if err != nil {
-		return
-	}
-
-	_, err = tx.Exec(`CREATE INDEX IF NOT EXISTS AddressesDeviceIDAddressIndex ON Addresses (DeviceID, Address)`)
-	return
-}
-
-func qlCompile(db *sql.DB) (map[string]*sql.Stmt, error) {
-	stmts := map[string]string{
-		"cleanAddress":  `DELETE FROM Addresses WHERE Seen < now() - duration("2h")`,
-		"cleanDevice":   fmt.Sprintf(`DELETE FROM Devices WHERE Seen < now() - duration("%dh")`, maxDeviceAge/3600),
-		"countAddress":  "SELECT count(*) FROM Addresses",
-		"countDevice":   "SELECT count(*) FROM Devices",
-		"insertAddress": "INSERT INTO Addresses (DeviceID, Seen, Address) VALUES ($1, now(), $2)",
-		"insertDevice":  "INSERT INTO Devices (DeviceID, Seen) VALUES ($1, now())",
-		"selectAddress": `SELECT Address from Addresses WHERE DeviceID==$1 AND Seen > now() - duration("1h") LIMIT 16`,
-		"selectDevice":  "SELECT Seen FROM Devices WHERE DeviceID==$1",
-		"updateAddress": "UPDATE Addresses Seen=now() WHERE DeviceID==$1 AND Address==$2",
-		"updateDevice":  "UPDATE Devices Seen=now() WHERE DeviceID==$1",
-	}
-
-	res := make(map[string]*sql.Stmt, len(stmts))
-	for key, stmt := range stmts {
-		prep, err := db.Prepare(stmt)
-		if err != nil {
-			log.Println("Failed to compile", stmt)
-			return nil, err
-		}
-		res[key] = prep
-	}
-	return res, nil
-}

cmd/stdiscosrv/querysrv.go

@@ -1,492 +0,0 @@
-// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-
-package main
-
-import (
-	"bytes"
-	"crypto/tls"
-	"database/sql"
-	"encoding/json"
-	"encoding/pem"
-	"fmt"
-	"log"
-	"math/rand"
-	"net"
-	"net/http"
-	"net/url"
-	"strconv"
-	"sync"
-	"time"
-
-	"github.com/golang/groupcache/lru"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"golang.org/x/net/context"
-	"golang.org/x/time/rate"
-)
-
-type querysrv struct {
-	addr     string
-	db       *sql.DB
-	prep     map[string]*sql.Stmt
-	limiter  *safeCache
-	cert     tls.Certificate
-	listener net.Listener
-}
-
-type announcement struct {
-	Seen      time.Time `json:"seen"`
-	Addresses []string  `json:"addresses"`
-}
-
-type safeCache struct {
-	*lru.Cache
-	mut sync.Mutex
-}
-
-func (s *safeCache) Get(key string) (val interface{}, ok bool) {
-	s.mut.Lock()
-	val, ok = s.Cache.Get(key)
-	s.mut.Unlock()
-	return
-}
-
-func (s *safeCache) Add(key string, val interface{}) {
-	s.mut.Lock()
-	s.Cache.Add(key, val)
-	s.mut.Unlock()
-}
-
-type requestID int64
-
-func (i requestID) String() string {
-	return fmt.Sprintf("%016x", int64(i))
-}
-
-type contextKey int
-
-const idKey contextKey = iota
-
-func negCacheFor(lastSeen time.Time) int {
-	since := time.Since(lastSeen).Seconds()
-	if since >= maxDeviceAge {
-		return maxNegCache
-	}
-	if since < 0 {
-		// That's weird
-		return minNegCache
-	}
-
-	// Return a value linearly scaled from minNegCache (at zero seconds ago)
-	// to maxNegCache (at maxDeviceAge seconds ago).
-	r := since / maxDeviceAge
-	return int(minNegCache + r*(maxNegCache-minNegCache))
-}
-
-func (s *querysrv) Serve() {
-	s.limiter = &safeCache{
-		Cache: lru.New(lruSize),
-	}
-
-	if useHTTP {
-		listener, err := net.Listen("tcp", s.addr)
-		if err != nil {
-			log.Println("Listen:", err)
-			return
-		}
-		s.listener = listener
-	} else {
-		tlsCfg := &tls.Config{
-			Certificates:           []tls.Certificate{s.cert},
-			ClientAuth:             tls.RequestClientCert,
-			SessionTicketsDisabled: true,
-			MinVersion:             tls.VersionTLS12,
-			CipherSuites: []uint16{
-				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
-				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
-				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
-				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
-				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
-			},
-		}
-
-		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
-		if err != nil {
-			log.Println("Listen:", err)
-			return
-		}
-		s.listener = tlsListener
-	}
-
-	http.HandleFunc("/v2/", s.handler)
-	http.HandleFunc("/ping", handlePing)
-
-	srv := &http.Server{
-		ReadTimeout:    5 * time.Second,
-		WriteTimeout:   5 * time.Second,
-		MaxHeaderBytes: 1 << 10,
-	}
-
-	if err := srv.Serve(s.listener); err != nil {
-		log.Println("Serve:", err)
-	}
-}
-
-var topCtx = context.Background()
-
-func (s *querysrv) handler(w http.ResponseWriter, req *http.Request) {
-	reqID := requestID(rand.Int63())
-	ctx := context.WithValue(topCtx, idKey, reqID)
-
-	if debug {
-		log.Println(reqID, req.Method, req.URL)
-	}
-
-	t0 := time.Now()
-	defer func() {
-		diff := time.Since(t0)
-		var comment string
-		if diff > time.Second {
-			comment = "(very slow request)"
-		} else if diff > 100*time.Millisecond {
-			comment = "(slow request)"
-		}
-		if comment != "" || debug {
-			log.Println(reqID, req.Method, req.URL, "completed in", diff, comment)
-		}
-	}()
-
-	var remoteIP net.IP
-	if useHTTP {
-		remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
-	} else {
-		addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
-		if err != nil {
-			log.Println("remoteAddr:", err)
-			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-			return
-		}
-		remoteIP = addr.IP
-	}
-
-	if s.limit(remoteIP) {
-		if debug {
-			log.Println(remoteIP, "is limited")
-		}
-		w.Header().Set("Retry-After", "60")
-		http.Error(w, "Too Many Requests", 429)
-		return
-	}
-
-	switch req.Method {
-	case "GET":
-		s.handleGET(ctx, w, req)
-	case "POST":
-		s.handlePOST(ctx, remoteIP, w, req)
-	default:
-		globalStats.Error()
-		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
-	}
-}
-
-func (s *querysrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
-	reqID := ctx.Value(idKey).(requestID)
-
-	deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
-	if err != nil {
-		if debug {
-			log.Println(reqID, "bad device param")
-		}
-		globalStats.Error()
-		http.Error(w, "Bad Request", http.StatusBadRequest)
-		return
-	}
-
-	var ann announcement
-
-	ann.Seen, err = s.getDeviceSeen(deviceID)
-	negCache := strconv.Itoa(negCacheFor(ann.Seen))
-	w.Header().Set("Retry-After", negCache)
-	w.Header().Set("Cache-Control", "public, max-age="+negCache)
-
-	if err != nil {
-		// The device is not in the database.
-		globalStats.Query()
-		http.Error(w, "Not Found", http.StatusNotFound)
-		return
-	}
-
-	t0 := time.Now()
-	ann.Addresses, err = s.getAddresses(ctx, deviceID)
-	if err != nil {
-		log.Println(reqID, "getAddresses:", err)
-		globalStats.Error()
-		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-		return
-	}
-	if debug {
-		log.Println(reqID, "getAddresses in", time.Since(t0))
-	}
-
-	globalStats.Query()
-
-	if len(ann.Addresses) == 0 {
-		http.Error(w, "Not Found", http.StatusNotFound)
-		return
-	}
-
-	globalStats.Answer()
-
-	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(ann)
-}
-
-func (s *querysrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
-	reqID := ctx.Value(idKey).(requestID)
-
-	rawCert := certificateBytes(req)
-	if rawCert == nil {
-		if debug {
-			log.Println(reqID, "no certificates")
-		}
-		globalStats.Error()
-		http.Error(w, "Forbidden", http.StatusForbidden)
-		return
-	}
-
-	var ann announcement
-	if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
-		if debug {
-			log.Println(reqID, "decode:", err)
-		}
-		globalStats.Error()
-		http.Error(w, "Bad Request", http.StatusBadRequest)
-		return
-	}
-
-	deviceID := protocol.NewDeviceID(rawCert)
-
-	// handleAnnounce returns *two* errors. The first indicates a problem with
-	// something the client posted to us. We should return a 400 Bad Request
-	// and not worry about it. The second indicates that the request was fine,
-	// but something internal messed up. We should log it and respond with a
-	// more apologetic 500 Internal Server Error.
-	userErr, internalErr := s.handleAnnounce(ctx, remoteIP, deviceID, ann.Addresses)
-	if userErr != nil {
-		if debug {
-			log.Println(reqID, "handleAnnounce:", userErr)
-		}
-		globalStats.Error()
-		http.Error(w, "Bad Request", http.StatusBadRequest)
-		return
-	}
-	if internalErr != nil {
-		log.Println(reqID, "handleAnnounce:", internalErr)
-		globalStats.Error()
-		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-		return
-	}
-
-	globalStats.Announce()
-
-	// TODO: Slowly increase this for stable clients
-	w.Header().Set("Reannounce-After", "1800")
-
-	// We could return the lookup result here, but it's kind of unnecessarily
-	// expensive to go query the database again so we let the client decide to
-	// do a lookup if they really care.
-	w.WriteHeader(http.StatusNoContent)
-}
-
-func (s *querysrv) Stop() {
-	s.listener.Close()
-}
-
-func (s *querysrv) handleAnnounce(ctx context.Context, remote net.IP, deviceID protocol.DeviceID, addresses []string) (userErr, internalErr error) {
-	reqID := ctx.Value(idKey).(requestID)
-
-	tx, err := s.db.Begin()
-	if err != nil {
-		internalErr = err
-		return
-	}
-
-	defer func() {
-		// Since we return from a bunch of different places, we handle
-		// rollback in the defer.
-		if internalErr != nil || userErr != nil {
-			tx.Rollback()
-		}
-	}()
-
-	for _, annAddr := range addresses {
-		uri, err := url.Parse(annAddr)
-		if err != nil {
-			userErr = err
-			return
-		}
-
-		host, port, err := net.SplitHostPort(uri.Host)
-		if err != nil {
-			userErr = err
-			return
-		}
-
-		ip := net.ParseIP(host)
-		if host == "" || ip.IsUnspecified() {
-			// Do not use IPv6 remote address if requested scheme is tcp4
-			if uri.Scheme == "tcp4" && remote.To4() == nil {
-				continue
-			}
-
-			// Do not use IPv4 remote address if requested scheme is tcp6
-			if uri.Scheme == "tcp6" && remote.To4() != nil {
-				continue
-			}
-
-			host = remote.String()
-		}
-
-		uri.Host = net.JoinHostPort(host, port)
-
-		if err := s.updateAddress(ctx, tx, deviceID, uri.String()); err != nil {
-			internalErr = err
-			return
-		}
-	}
-
-	if err := s.updateDevice(ctx, tx, deviceID); err != nil {
-		internalErr = err
-		return
-	}
-
-	t0 := time.Now()
-	internalErr = tx.Commit()
-	if debug {
-		log.Println(reqID, "commit in", time.Since(t0))
-	}
-	return
-}
-
-func (s *querysrv) limit(remote net.IP) bool {
-	key := remote.String()
-
-	bkt, ok := s.limiter.Get(key)
-	if ok {
-		bkt := bkt.(*rate.Limiter)
-		if !bkt.Allow() {
-			// Rate limit exceeded; ignore packet
-			return true
-		}
-	} else {
-		// limitAvg is in packets per ten seconds.
-		s.limiter.Add(key, rate.NewLimiter(rate.Limit(limitAvg)/10, limitBurst))
-	}
-
-	return false
-}
-
-func (s *querysrv) updateDevice(ctx context.Context, tx *sql.Tx, device protocol.DeviceID) error {
-	reqID := ctx.Value(idKey).(requestID)
-	t0 := time.Now()
-	res, err := tx.Stmt(s.prep["updateDevice"]).Exec(device.String())
-	if err != nil {
-		return err
-	}
-	if debug {
-		log.Println(reqID, "updateDevice in", time.Since(t0))
-	}
-
-	if rows, _ := res.RowsAffected(); rows == 0 {
-		t0 = time.Now()
-		_, err := tx.Stmt(s.prep["insertDevice"]).Exec(device.String())
-		if err != nil {
-			return err
-		}
-		if debug {
-			log.Println(reqID, "insertDevice in", time.Since(t0))
-		}
-	}
-
-	return nil
-}
-
-func (s *querysrv) updateAddress(ctx context.Context, tx *sql.Tx, device protocol.DeviceID, uri string) error {
-	res, err := tx.Stmt(s.prep["updateAddress"]).Exec(device.String(), uri)
-	if err != nil {
-		return err
-	}
-
-	if rows, _ := res.RowsAffected(); rows == 0 {
-		_, err := tx.Stmt(s.prep["insertAddress"]).Exec(device.String(), uri)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (s *querysrv) getAddresses(ctx context.Context, device protocol.DeviceID) ([]string, error) {
-	rows, err := s.prep["selectAddress"].Query(device.String())
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	var res []string
-	for rows.Next() {
-		var addr string
-
-		err := rows.Scan(&addr)
-		if err != nil {
-			log.Println("Scan:", err)
-			continue
-		}
-		res = append(res, addr)
-	}
-
-	return res, nil
-}
-
-func (s *querysrv) getDeviceSeen(device protocol.DeviceID) (time.Time, error) {
-	row := s.prep["selectDevice"].QueryRow(device.String())
-	var seen time.Time
-	if err := row.Scan(&seen); err != nil {
-		return time.Time{}, err
-	}
-	return seen.In(time.UTC), nil
-}
-
-func handlePing(w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(204)
-}
-
-func certificateBytes(req *http.Request) []byte {
-	if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
-		return req.TLS.PeerCertificates[0].Raw
-	}
-
-	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
-		bs := []byte(hdr)
-		// The certificate is in PEM format but with spaces for newlines. We
-		// need to reinstate the newlines for the PEM decoder. But we need to
-		// leave the spaces in the BEGIN and END lines - the first and last
-		// space - alone.
-		firstSpace := bytes.Index(bs, []byte(" "))
-		lastSpace := bytes.LastIndex(bs, []byte(" "))
-		for i := firstSpace + 1; i < lastSpace; i++ {
-			if bs[i] == ' ' {
-				bs[i] = '\n'
-			}
-		}
-		block, _ := pem.Decode(bs)
-		if block == nil {
-			// Decoding failed
-			return nil
-		}
-		return block.Bytes
-	}
-
-	return nil
-}

cmd/stdiscosrv/replication.go

@@ -0,0 +1,315 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/binary"
+	"fmt"
+	io "io"
+	"log"
+	"net"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+const replicationReadTimeout = time.Minute
+const replicationHeartbeatInterval = time.Second * 30
+
+type replicator interface {
+	send(key string, addrs []DatabaseAddress, seen int64)
+}
+
+// a replicationSender tries to connect to the remote address and provide
+// them with a feed of replication updates.
+type replicationSender struct {
+	dst        string
+	cert       tls.Certificate // our certificate
+	allowedIDs []protocol.DeviceID
+	outbox     chan ReplicationRecord
+}
+
+func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
+	return &replicationSender{
+		dst:        dst,
+		cert:       cert,
+		allowedIDs: allowedIDs,
+		outbox:     make(chan ReplicationRecord, replicationOutboxSize),
+	}
+}
+
+func (s *replicationSender) Serve(ctx context.Context) error {
+	// Sleep a little at startup. Peers often restart at the same time, and
+	// this avoid the service failing and entering backoff state
+	// unnecessarily, while also reducing the reconnect rate to something
+	// reasonable by default.
+	time.Sleep(2 * time.Second)
+
+	tlsCfg := &tls.Config{
+		Certificates:       []tls.Certificate{s.cert},
+		MinVersion:         tls.VersionTLS12,
+		InsecureSkipVerify: true,
+	}
+
+	// Dial the TLS connection.
+	conn, err := tls.Dial("tcp", s.dst, tlsCfg)
+	if err != nil {
+		log.Println("Replication connect:", err)
+		return err
+	}
+	defer func() {
+		conn.SetWriteDeadline(time.Now().Add(time.Second))
+		conn.Close()
+	}()
+
+	// Get the other side device ID.
+	remoteID, err := deviceID(conn)
+	if err != nil {
+		log.Println("Replication connect:", err)
+		return err
+	}
+
+	// Verify it's in the set of allowed device IDs.
+	if !deviceIDIn(remoteID, s.allowedIDs) {
+		log.Println("Replication connect: unexpected device ID:", remoteID)
+		return err
+	}
+
+	heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
+	defer heartBeatTicker.Stop()
+
+	// Send records.
+	buf := make([]byte, 1024)
+	for {
+		select {
+		case <-heartBeatTicker.C:
+			if len(s.outbox) > 0 {
+				// No need to send heartbeats if there are events/prevrious
+				// heartbeats to send, they will keep the connection alive.
+				continue
+			}
+			// Empty replication message is the heartbeat:
+			s.outbox <- ReplicationRecord{}
+
+		case rec := <-s.outbox:
+			// Buffer must hold record plus four bytes for size
+			size := rec.Size()
+			if len(buf) < size+4 {
+				buf = make([]byte, size+4)
+			}
+
+			// Record comes after the four bytes size
+			n, err := rec.MarshalTo(buf[4:])
+			if err != nil {
+				// odd to get an error here, but we haven't sent anything
+				// yet so it's not fatal
+				replicationSendsTotal.WithLabelValues("error").Inc()
+				log.Println("Replication marshal:", err)
+				continue
+			}
+			binary.BigEndian.PutUint32(buf, uint32(n))
+
+			// Send
+			conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
+			if _, err := conn.Write(buf[:4+n]); err != nil {
+				replicationSendsTotal.WithLabelValues("error").Inc()
+				log.Println("Replication write:", err)
+				// Yes, we are losing the replication event here.
+				return err
+			}
+			replicationSendsTotal.WithLabelValues("success").Inc()
+
+		case <-ctx.Done():
+			return nil
+		}
+	}
+}
+
+func (s *replicationSender) String() string {
+	return fmt.Sprintf("replicationSender(%q)", s.dst)
+}
+
+func (s *replicationSender) send(key string, ps []DatabaseAddress, _ int64) {
+	item := ReplicationRecord{
+		Key:       key,
+		Addresses: ps,
+	}
+
+	// The send should never block. The inbox is suitably buffered for at
+	// least a few seconds of stalls, which shouldn't happen in practice.
+	select {
+	case s.outbox <- item:
+	default:
+		replicationSendsTotal.WithLabelValues("drop").Inc()
+	}
+}
+
+// a replicationMultiplexer sends to multiple replicators
+type replicationMultiplexer []replicator
+
+func (m replicationMultiplexer) send(key string, ps []DatabaseAddress, seen int64) {
+	for _, s := range m {
+		// each send is nonblocking
+		s.send(key, ps, seen)
+	}
+}
+
+// replicationListener accepts incoming connections and reads replication
+// items from them. Incoming items are applied to the KV store.
+type replicationListener struct {
+	addr       string
+	cert       tls.Certificate
+	allowedIDs []protocol.DeviceID
+	db         database
+}
+
+func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
+	return &replicationListener{
+		addr:       addr,
+		cert:       cert,
+		allowedIDs: allowedIDs,
+		db:         db,
+	}
+}
+
+func (l *replicationListener) Serve(ctx context.Context) error {
+	tlsCfg := &tls.Config{
+		Certificates:       []tls.Certificate{l.cert},
+		ClientAuth:         tls.RequestClientCert,
+		MinVersion:         tls.VersionTLS12,
+		InsecureSkipVerify: true,
+	}
+
+	lst, err := tls.Listen("tcp", l.addr, tlsCfg)
+	if err != nil {
+		log.Println("Replication listen:", err)
+		return err
+	}
+	defer lst.Close()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+		}
+
+		// Accept a connection
+		conn, err := lst.Accept()
+		if err != nil {
+			log.Println("Replication accept:", err)
+			return err
+		}
+
+		// Figure out the other side device ID
+		remoteID, err := deviceID(conn.(*tls.Conn))
+		if err != nil {
+			log.Println("Replication accept:", err)
+			conn.SetWriteDeadline(time.Now().Add(time.Second))
+			conn.Close()
+			continue
+		}
+
+		// Verify it is in the set of allowed device IDs
+		if !deviceIDIn(remoteID, l.allowedIDs) {
+			log.Println("Replication accept: unexpected device ID:", remoteID)
+			conn.SetWriteDeadline(time.Now().Add(time.Second))
+			conn.Close()
+			continue
+		}
+
+		go l.handle(ctx, conn)
+	}
+}
+
+func (l *replicationListener) String() string {
+	return fmt.Sprintf("replicationListener(%q)", l.addr)
+}
+
+func (l *replicationListener) handle(ctx context.Context, conn net.Conn) {
+	defer func() {
+		conn.SetWriteDeadline(time.Now().Add(time.Second))
+		conn.Close()
+	}()
+
+	buf := make([]byte, 1024)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		conn.SetReadDeadline(time.Now().Add(replicationReadTimeout))
+
+		// First four bytes are the size
+		if _, err := io.ReadFull(conn, buf[:4]); err != nil {
+			log.Println("Replication read size:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			return
+		}
+
+		// Read the rest of the record
+		size := int(binary.BigEndian.Uint32(buf[:4]))
+		if len(buf) < size {
+			buf = make([]byte, size)
+		}
+
+		if size == 0 {
+			// Heartbeat, ignore
+			continue
+		}
+
+		if _, err := io.ReadFull(conn, buf[:size]); err != nil {
+			log.Println("Replication read record:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			return
+		}
+
+		// Unmarshal
+		var rec ReplicationRecord
+		if err := rec.Unmarshal(buf[:size]); err != nil {
+			log.Println("Replication unmarshal:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			continue
+		}
+
+		// Store
+		l.db.merge(rec.Key, rec.Addresses, rec.Seen)
+		replicationRecvsTotal.WithLabelValues("success").Inc()
+	}
+}
+
+func deviceID(conn *tls.Conn) (protocol.DeviceID, error) {
+	// Handshake may not be complete on the server side yet, which we need
+	// to get the client certificate.
+	if !conn.ConnectionState().HandshakeComplete {
+		if err := conn.Handshake(); err != nil {
+			return protocol.DeviceID{}, err
+		}
+	}
+
+	// We expect exactly one certificate.
+	certs := conn.ConnectionState().PeerCertificates
+	if len(certs) != 1 {
+		return protocol.DeviceID{}, fmt.Errorf("unexpected number of certificates (%d != 1)", len(certs))
+	}
+
+	return protocol.NewDeviceID(certs[0].Raw), nil
+}
+
+func deviceIDIn(id protocol.DeviceID, ids []protocol.DeviceID) bool {
+	for _, candidate := range ids {
+		if id == candidate {
+			return true
+		}
+	}
+	return false
+}

cmd/stdiscosrv/scripts/preinst

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+addgroup --system syncthing
+adduser --system --home /var/lib/syncthing-discosrv --ingroup syncthing syncthing-discosrv

cmd/stdiscosrv/stats.go

@@ -1,141 +1,124 @@
-// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
-	"bytes"
-	"database/sql"
-	"fmt"
-	"io/ioutil"
-	"log"
 	"os"
-	"sync/atomic"
-	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
 )
 
-type stats struct {
-	// Incremented atomically
-	announces int64
-	queries   int64
-	answers   int64
-	errors    int64
-}
+var (
+	apiRequestsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "api_requests_total",
+			Help:      "Number of API requests.",
+		}, []string{"type", "result"})
+	apiRequestsSeconds = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Namespace:  "syncthing",
+			Subsystem:  "discovery",
+			Name:       "api_requests_seconds",
+			Help:       "Latency of API requests.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		}, []string{"type"})
 
-func (s *stats) Announce() {
-	atomic.AddInt64(&s.announces, 1)
-}
+	lookupRequestsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "lookup_requests_total",
+			Help:      "Number of lookup requests.",
+		}, []string{"result"})
+	announceRequestsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "announcement_requests_total",
+			Help:      "Number of announcement requests.",
+		}, []string{"result"})
 
-func (s *stats) Query() {
-	atomic.AddInt64(&s.queries, 1)
-}
+	replicationSendsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "replication_sends_total",
+			Help:      "Number of replication sends.",
+		}, []string{"result"})
+	replicationRecvsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "replication_recvs_total",
+			Help:      "Number of replication receives.",
+		}, []string{"result"})
 
-func (s *stats) Answer() {
-	atomic.AddInt64(&s.answers, 1)
-}
+	databaseKeys = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "database_keys",
+			Help:      "Number of database keys at last count.",
+		}, []string{"category"})
+	databaseStatisticsSeconds = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "database_statistics_seconds",
+			Help:      "Time spent running the statistics routine.",
+		})
 
-func (s *stats) Error() {
-	atomic.AddInt64(&s.errors, 1)
-}
+	databaseOperations = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "discovery",
+			Name:      "database_operations_total",
+			Help:      "Number of database operations.",
+		}, []string{"operation", "result"})
+	databaseOperationSeconds = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Namespace:  "syncthing",
+			Subsystem:  "discovery",
+			Name:       "database_operation_seconds",
+			Help:       "Latency of database operations.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		}, []string{"operation"})
+)
 
-// Reset returns a copy of the current stats and resets the counters to
-// zero.
-func (s *stats) Reset() stats {
-	// Create a copy of the stats using atomic reads
-	copy := stats{
-		announces: atomic.LoadInt64(&s.announces),
-		queries:   atomic.LoadInt64(&s.queries),
-		answers:   atomic.LoadInt64(&s.answers),
-		errors:    atomic.LoadInt64(&s.errors),
+const (
+	dbOpGet             = "get"
+	dbOpPut             = "put"
+	dbOpMerge           = "merge"
+	dbOpDelete          = "delete"
+	dbResSuccess        = "success"
+	dbResNotFound       = "not_found"
+	dbResError          = "error"
+	dbResUnmarshalError = "unmarsh_err"
+)
+
+func init() {
+	prometheus.MustRegister(apiRequestsTotal, apiRequestsSeconds,
+		lookupRequestsTotal, announceRequestsTotal,
+		replicationSendsTotal, replicationRecvsTotal,
+		databaseKeys, databaseStatisticsSeconds,
+		databaseOperations, databaseOperationSeconds)
+
+	processCollectorOpts := collectors.ProcessCollectorOpts{
+		Namespace: "syncthing_discovery",
+		PidFn: func() (int, error) {
+			return os.Getpid(), nil
+		},
 	}
 
-	// Reset the stats by subtracting the values that we copied
-	atomic.AddInt64(&s.announces, -copy.announces)
-	atomic.AddInt64(&s.queries, -copy.queries)
-	atomic.AddInt64(&s.answers, -copy.answers)
-	atomic.AddInt64(&s.errors, -copy.errors)
+	prometheus.MustRegister(
+		collectors.NewProcessCollector(processCollectorOpts),
+	)
 
-	return copy
-}
-
-type statssrv struct {
-	intv time.Duration
-	file string
-	db   *sql.DB
-}
-
-func (s *statssrv) Serve() {
-	lastReset := time.Now()
-	for {
-		time.Sleep(next(s.intv))
-
-		stats := globalStats.Reset()
-		d := time.Since(lastReset).Seconds()
-		lastReset = time.Now()
-
-		log.Printf("Stats: %.02f announces/s, %.02f queries/s, %.02f answers/s, %.02f errors/s",
-			float64(stats.announces)/d, float64(stats.queries)/d, float64(stats.answers)/d, float64(stats.errors)/d)
-
-		if s.file != "" {
-			s.writeToFile(stats, d)
-		}
-	}
-}
-
-func (s *statssrv) Stop() {
-	panic("stop unimplemented")
-}
-
-func (s *statssrv) writeToFile(stats stats, secs float64) {
-	newLine := []byte("\n")
-
-	var addrs int
-	row := s.db.QueryRow("SELECT COUNT(*) FROM Addresses")
-	if err := row.Scan(&addrs); err != nil {
-		log.Println("stats query:", err)
-		return
-	}
-
-	fd, err := os.OpenFile(s.file, os.O_RDWR|os.O_CREATE, 0666)
-	if err != nil {
-		log.Println("stats file:", err)
-		return
-	}
-	defer func() {
-		err = fd.Close()
-		if err != nil {
-			log.Println("stats file:", err)
-		}
-	}()
-
-	bs, err := ioutil.ReadAll(fd)
-	if err != nil {
-		log.Println("stats file:", err)
-		return
-	}
-	lines := bytes.Split(bytes.TrimSpace(bs), newLine)
-	if len(lines) > 12 {
-		lines = lines[len(lines)-12:]
-	}
-
-	latest := fmt.Sprintf("%v: %6d addresses, %8.02f announces/s, %8.02f queries/s, %8.02f answers/s, %8.02f errors/s\n",
-		time.Now().UTC().Format(time.RFC3339), addrs,
-		float64(stats.announces)/secs, float64(stats.queries)/secs, float64(stats.answers)/secs, float64(stats.errors)/secs)
-	lines = append(lines, []byte(latest))
-
-	_, err = fd.Seek(0, 0)
-	if err != nil {
-		log.Println("stats file:", err)
-		return
-	}
-	err = fd.Truncate(0)
-	if err != nil {
-		log.Println("stats file:", err)
-		return
-	}
-
-	_, err = fd.Write(bytes.Join(lines, newLine))
-	if err != nil {
-		log.Println("stats file:", err)
-		return
-	}
 }

cmd/stevents/main.go

@@ -28,16 +28,21 @@ func main() {
 	log.SetFlags(0)
 
 	target := flag.String("target", "localhost:8384", "Target Syncthing instance")
+	types := flag.String("types", "", "Filter for specific event types (comma-separated)")
 	apikey := flag.String("apikey", "", "Syncthing API key")
 	flag.Parse()
 
 	if *apikey == "" {
 		log.Fatal("Must give -apikey argument")
 	}
+	var eventsArg string
+	if len(*types) > 0 {
+		eventsArg = "&events=" + *types
+	}
 
 	since := 0
 	for {
-		req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d", *target, since), nil)
+		req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d%s", *target, since, eventsArg), nil)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/stfileinfo/main.go

@@ -68,8 +68,8 @@ func main() {
 		}
 
 		blockSize := int(fi.Size())
-		if *standardBlocks || blockSize < protocol.BlockSize {
-			blockSize = protocol.BlockSize
+		if *standardBlocks || blockSize < protocol.MinBlockSize {
+			blockSize = protocol.BlockSize(fi.Size())
 		}
 		bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
 		if err != nil {

cmd/stfinddevice/main.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"errors"
 	"flag"
@@ -17,6 +18,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -82,7 +84,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 }
 
 func checkServer(deviceID protocol.DeviceID, server string) checkResult {
-	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger, nil)
 	if err != nil {
 		return checkResult{error: err}
 	}
@@ -94,7 +96,7 @@ func checkServer(deviceID protocol.DeviceID, server string) checkResult {
 	})
 
 	go func() {
-		addresses, err := disco.Lookup(deviceID)
+		addresses, err := disco.Lookup(context.Background(), deviceID)
 		res <- checkResult{addresses: addresses, error: err}
 	}()
 

cmd/stfindignored/main.go

@@ -0,0 +1,44 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Command stfindignored lists ignored files under a given folder root.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/syncthing/syncthing/lib/fs"
+	"github.com/syncthing/syncthing/lib/ignore"
+)
+
+func main() {
+	flag.Parse()
+	root := flag.Arg(0)
+	if root == "" {
+		root = "."
+	}
+
+	vfs := fs.NewWalkFilesystem(fs.NewFilesystem(fs.FilesystemTypeBasic, root))
+
+	ign := ignore.New(vfs)
+	if err := ign.Load(".stignore"); err != nil {
+		fmt.Fprintf(os.Stderr, "Fatal: loading ignores: %v\n", err)
+		os.Exit(1)
+	}
+
+	vfs.Walk(".", func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Warning: %s: %v\n", path, err)
+			return fs.SkipDir
+		}
+		if ign.Match(path).IsIgnored() {
+			fmt.Println(path)
+		}
+		return nil
+	})
+}

cmd/stgenfiles/main.go

@@ -82,7 +82,7 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
 		return err
 	}
 
-	_ = os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
+	os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
 
 	t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
 	return os.Chtimes(p1, t, t)

cmd/stindex/dump.go

@@ -1,83 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"encoding/binary"
-	"fmt"
-	"log"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-func dump(ldb *db.Instance) {
-	it := ldb.NewIterator(nil, nil)
-	for it.Next() {
-		key := it.Key()
-		switch key[0] {
-		case db.KeyTypeDevice:
-			folder := binary.BigEndian.Uint32(key[1:])
-			device := binary.BigEndian.Uint32(key[1+4:])
-			name := nulString(key[1+4+4:])
-			fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
-
-			var f protocol.FileInfo
-			err := f.Unmarshal(it.Value())
-			if err != nil {
-				log.Fatal(err)
-			}
-			fmt.Printf(" V:%v\n", f)
-
-		case db.KeyTypeGlobal:
-			folder := binary.BigEndian.Uint32(key[1:])
-			name := nulString(key[1+4:])
-			var flv db.VersionList
-			flv.Unmarshal(it.Value())
-			fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
-
-		case db.KeyTypeBlock:
-			folder := binary.BigEndian.Uint32(key[1:])
-			hash := key[1+4 : 1+4+32]
-			name := nulString(key[1+4+32:])
-			fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
-
-		case db.KeyTypeDeviceStatistic:
-			fmt.Printf("[dstat] K:%x V:%x\n", it.Key(), it.Value())
-
-		case db.KeyTypeFolderStatistic:
-			fmt.Printf("[fstat] K:%x V:%x\n", it.Key(), it.Value())
-
-		case db.KeyTypeVirtualMtime:
-			folder := binary.BigEndian.Uint32(key[1:])
-			name := nulString(key[1+4:])
-			val := it.Value()
-			var real, virt time.Time
-			real.UnmarshalBinary(val[:len(val)/2])
-			virt.UnmarshalBinary(val[len(val)/2:])
-			fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, real, virt)
-
-		case db.KeyTypeFolderIdx:
-			key := binary.BigEndian.Uint32(it.Key()[1:])
-			fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
-
-		case db.KeyTypeDeviceIdx:
-			key := binary.BigEndian.Uint32(it.Key()[1:])
-			val := it.Value()
-			if len(val) == 0 {
-				fmt.Printf("[deviceidx] K:%d V:<nil>\n", key)
-			} else {
-				dev := protocol.DeviceIDFromBytes(val)
-				fmt.Printf("[deviceidx] K:%d V:%s\n", key, dev)
-			}
-
-		default:
-			fmt.Printf("[???]\n  %x\n  %x\n", it.Key(), it.Value())
-		}
-	}
-}

cmd/stindex/main.go

@@ -1,47 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-
-	"github.com/syncthing/syncthing/lib/db"
-)
-
-func main() {
-	var mode string
-	log.SetFlags(0)
-	log.SetOutput(os.Stdout)
-
-	flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize")
-
-	flag.Parse()
-
-	path := flag.Arg(0)
-	if path == "" {
-		path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
-	}
-
-	fmt.Println("Path:", path)
-
-	ldb, err := db.Open(path)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if mode == "dump" {
-		dump(ldb)
-	} else if mode == "dumpsize" {
-		dumpsize(ldb)
-	} else {
-		fmt.Println("Unknown mode")
-	}
-}

cmd/stindex/util.go

@@ -1,52 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"log"
-	"os"
-	"path/filepath"
-	"runtime"
-
-	"github.com/syncthing/syncthing/lib/fs"
-)
-
-func nulString(bs []byte) string {
-	for i := range bs {
-		if bs[i] == 0 {
-			return string(bs[:i])
-		}
-	}
-	return string(bs)
-}
-
-func defaultConfigDir() string {
-	switch runtime.GOOS {
-	case "windows":
-		if p := os.Getenv("LocalAppData"); p != "" {
-			return filepath.Join(p, "Syncthing")
-		}
-		return filepath.Join(os.Getenv("AppData"), "Syncthing")
-
-	case "darwin":
-		dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
-		if err != nil {
-			log.Fatal(err)
-		}
-		return dir
-
-	default:
-		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
-			return filepath.Join(xdgCfg, "syncthing")
-		}
-		dir, err := fs.ExpandTilde("~/.config/syncthing")
-		if err != nil {
-			log.Fatal(err)
-		}
-		return dir
-	}
-}

cmd/strelaypoolsrv/auto/.gitignore

@@ -1 +1 @@
-gui.go
+gui.files.go

cmd/strelaypoolsrv/auto/doc.go

@@ -0,0 +1,10 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//go:generate go run ../../../script/genassets.go -o gui.files.go ../gui
+
+// Package auto contains auto generated files for web assets.
+package auto

cmd/strelaypoolsrv/auto/noassets.go

@@ -1,17 +1,16 @@
-// Copyright (C) 2016 The Syncthing Authors.
+// Copyright (C) 2021 The Syncthing Authors.
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+//go:build noassets
+// +build noassets
 
-type mockedConnections struct{}
+package auto
 
-func (m *mockedConnections) Status() map[string]interface{} {
+import "github.com/syncthing/syncthing/lib/assets"
+
+func Assets() map[string]assets.Asset {
 	return nil
 }
-
-func (m *mockedConnections) NATType() string {
-	return ""
-}

cmd/strelaypoolsrv/gui/index.html

@@ -9,8 +9,15 @@
     <meta name="author" content=""/>
 
     <title>Relay stats</title>
-    <link href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
-    <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/font-awesome/4.6.1/css/font-awesome.min.css"/>
+    <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"/>
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
+    <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"/>
+    <link rel="stylesheet" href="https://unpkg.com/leaflet@1.6.0/dist/leaflet.css"
+  integrity="sha512-xwE/Az9zrjBIphAcBb3F6JVqxf46+CDLwfLMHloNu6KEQCAWi6HcDUbeOfBIptF7tcCzusKFjFw2yuvEpDL9wQ=="
+  crossorigin=""/>
+  <script src="https://unpkg.com/leaflet@1.6.0/dist/leaflet.js"
+  integrity="sha512-gZwIG9x3wUXg2hdXF6+rVkLF/0Vi9U8D2Ntg4Ga5I5BZpVkVxlJWbSQtXPSiUTtC0TjtGOmxa1AJPuV0CPthew=="
+  crossorigin=""></script>
 
     <style>
       #map {
@@ -38,17 +45,19 @@
     <div class="container">
       <h1>Relay Pool Data</h1>
       <div ng-if="relays === undefined" class="text-center">
-        <img src="//cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
-        <p>Please wait while we gather data</p>
+        <img src="https://cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
+        <p>Please wait while we gather data…</p>
       </div>
       <div>
         <div ng-show="relays !== undefined" class="ng-hide">
           <p>
-            Currently {{ relays.length }} relays online ({{ totals.goMaxProcs }} cores in total).
+            The relays listed on this page are not managed or vetted by the Syncthing project.
+            Each relay is the responsibility of the relay operator.
+            Currently {{ relays.length }} relays are online.
           </p>
         </div>
         <div id="map"></div> <!-- Can't hide the map, otherwise it freaks out -->
-        <p>The circle size represents how much bytes the relay transferred relative to other relays</p>
+        <p>The circle size represents how much bytes the relay has transferred relatively to other relays.</p>
       </div>
       <div>
         <table class="table table-striped table-condensed table">
@@ -56,83 +65,83 @@
             <tr>
               <th rowspan="2">Address</td>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.numActiveSessions'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.numActiveSessions'; sortReverse = !sortReverse">
                   Sessions
-                  <span ng-show="sortType == 'status.numActiveSessions' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.numActiveSessions' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.numActiveSessions' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.numActiveSessions' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.numConnections'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.numConnections'; sortReverse = !sortReverse">
                   Connections
-                  <span ng-show="sortType == 'status.numConnections' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.numConnections' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.numConnections' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.numConnections' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.bytesProxied'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.bytesProxied'; sortReverse = !sortReverse">
                   Data relayed
-                  <span ng-show="sortType == 'status.bytesProxied' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.bytesProxied' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.bytesProxied' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.bytesProxied' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th colspan="6" class="text-center">Transfer rate in the last period</th>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.uptimeSeconds'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.uptimeSeconds'; sortReverse = !sortReverse">
                   Uptime hours
-                  <span ng-show="sortType == 'status.uptimeSeconds' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.uptimeSeconds' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.uptimeSeconds' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.uptimeSeconds' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th rowspan="2">
-                <a ng-click="sortType = 'status.options[\'provided-by\'] || \'\''; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.options[\'provided-by\'] || \'\''; sortReverse = !sortReverse">
                   Provided by
-                  <span ng-show="sortType == 'status.options[\'provided-by\'] || \'\'' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.options[\'provided-by\'] || \'\'' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.options[\'provided-by\'] || \'\'' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.options[\'provided-by\'] || \'\'' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
             </tr>
             <tr>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[0]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[0]'; sortReverse = !sortReverse">
                   10s
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[0]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[0]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[0]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[0]' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[1]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[1]'; sortReverse = !sortReverse">
                   1m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[1]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[1]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[1]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[1]' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[2]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[2]'; sortReverse = !sortReverse">
                   5m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[2]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[2]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[2]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[2]' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[3]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[3]'; sortReverse = !sortReverse">
                   15m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[3]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[3]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[3]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[3]' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[4]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[4]'; sortReverse = !sortReverse">
                   30m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[4]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[4]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[4]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[4]' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
               <th>
-                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[5]'; sortReverse = !sortReverse">
+                <a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[5]'; sortReverse = !sortReverse">
                   60m
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[5]' && !sortReverse" class="fa fa-caret-down"></span>
-                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[5]' && sortReverse" class="fa fa-caret-up"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[5]' && !sortReverse" class="fas fa-caret-down"></span>
+                  <span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[5]' && sortReverse" class="fas fa-caret-up"></span>
                 </a>
               </th>
             </tr>
@@ -140,21 +149,21 @@
           <tbody>
             <tr ng-repeat="relay in relays | orderBy:sortType:sortReverse:sortCompare" ng-mouseover="relay.showMarker()" ng-mouseleave="relay.hideMarker()">
               <td>{{ relay.address }}</td>
-              <td ng-if="relay.status === undefined" colspan="11" class="text-center">Looking up...</td>
-              <td ng-if-start="relay.status !== undefined">{{ relay.status.numActiveSessions }}</td>
-              <td>{{ relay.status.numConnections }}</td>
-              <td>{{ relay.status.bytesProxied | bytes }}</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
-              <td>{{ relay.status.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
-              <td ng-if="relay.status.uptimeSeconds != undefined">{{ relay.status.uptimeSeconds/60/60 | number:0 }}</td>
-              <td ng-if="relay.status.uptimeSeconds == undefined"></td>
-              <td title="{{ relay.status.options['provided-by'] || '' }}" ng-if-end>
-                {{ relay.status.options['provided-by'] || '' | limitTo:50 }}
-                <span ng-if="(relay.status.options['provided-by'] || '').length > 50">&hellip;
+              <td ng-if="!relay.stats" colspan="11"></td>
+              <td ng-if-start="relay.stats">{{ relay.stats.numActiveSessions }}</td>
+              <td>{{ relay.stats.numConnections }}</td>
+              <td>{{ relay.stats.bytesProxied | bytes }}</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
+              <td>{{ relay.stats.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
+              <td ng-if="relay.stats.uptimeSeconds != undefined">{{ relay.stats.uptimeSeconds/60/60 | number:0 }}</td>
+              <td ng-if="relay.stats.uptimeSeconds == undefined"></td>
+              <td title="{{ relay.stats.options['provided-by'] || '' }}" ng-if-end>
+                {{ relay.stats.options['provided-by'] || '' | limitTo:50 }}
+                <span ng-if="(relay.stats.options['provided-by'] || '').length > 50">&hellip;
               </td>
             </tr>
           </tbody>
@@ -179,23 +188,22 @@
       <hr>
       <p>
         This product includes GeoLite2 data created by MaxMind, available from
-        <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
+        <a href="https://www.maxmind.com">https://www.maxmind.com</a>.
       </p>
     </div>
 
 
-    <script type="text/javascript" src="//code.jquery.com/jquery-2.1.4.min.js"></script>
-    <script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
-    <script type="text/javascript" src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
-    <script type="text/javascript" src="//maps.googleapis.com/maps/api/js"></script>
+    <script type="text/javascript" src="https://code.jquery.com/jquery-2.1.4.min.js"></script>
+    <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
+    <script type="text/javascript" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
   </body>
 
   <script>
   angular.module('syncthing', [
   ])
-  .config(function($httpProvider) {
+  .config(['$httpProvider', function($httpProvider) {
     $httpProvider.defaults.timeout = 5000;
-  })
+  }])
   .filter('bytes', function() {
     return function(bytes, precision) {
       if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) return '-';
@@ -228,23 +236,24 @@
       numProxies: 0,
       uptimeSeconds: 0,
     };
-    $scope.map = new google.maps.Map(document.getElementById('map'), {
-      zoom: 1,
-      mapTypeId: google.maps.MapTypeId.ROADMAP
-    });
-    $scope.mapBounds = new google.maps.LatLngBounds();
+    $scope.map = L.map('map').setView([40.90296, 1.90925], 2);
+    L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',
+    {
+      attribution: 'Leaflet',
+      maxZoom: 17
+    }).addTo($scope.map);
     $scope.tooltipTemplate = $('#infoTemplate').html();
     $scope.usedLocations = {};
-    $scope.sortType = 'status.numActiveSessions';
+    $scope.sortType = 'stats.numActiveSessions';
     $scope.sortReverse = true;
     $scope.sortCompare = function(a, b) {
       if (a.value == b.value) {
         return 0;
       }
-      if (a.type == "undefined") {
+      if (a.type == "undefined" || a.type == "null") {
         return -1;
       }
-      if (b.type == "undefined") {
+      if (b.type == "undefined" || b.type == "null") {
         return 1;
       }
       return a.value > b.value ? 1 : -1;
@@ -252,29 +261,36 @@
 
     $http.get("/endpoint").then(function(response) {
       $scope.relays = response.data.relays;
-      var promises = [];
-      angular.forEach($scope.relays, function(relay) {
 
+      angular.forEach($scope.relays, function(relay) {
         relay.uri = constructURI(relay.url);
         relay.address = relay.url.split('/')[2];
 
         addMarkerToMap(relay);
 
-        promises.push(getRelayStatus(relay));
+        if (relay.stats) {
+          angular.forEach($scope.totals, function(value, key) {
+            if (typeof $scope.totals[key] == 'number') {
+              $scope.totals[key] += relay.stats[key];
+            } else if (typeof $scope.totals[key] == 'object' && $scope.totals[key] instanceof Array) {
+              angular.forEach($scope.totals[key], function(value, index) {
+                $scope.totals[key][index] += relay.stats[key][index];
+              });
+            }
+          });
+        }
       });
 
-      // Can only add circles once we know the totals for transfers, which means
-      // we need to resolve all statuses.
-      $q.all(promises).then(function() {
-        angular.forEach($scope.relays, function(relay) {
-          if (relay.status) {
-            addCircleToMap(relay);
-          }
-        });
+      // After the totals were calculated, add circles.
+      angular.forEach($scope.relays, function(relay) {
+        if (relay.stats) {
+          addCircleToMap(relay);
+        }
       });
 
-      $scope.map.fitBounds($scope.mapBounds);
       if ($scope.relays.length == 1) {
+        //Center to only relay with zoom
+        $scope.map.panTo(new L.LatLng(relays[0].location.latitude, relays[0].location.longitude));
         $scope.map.setZoom(13);
       }
     });
@@ -294,75 +310,50 @@
 
         var locParts = loc.split(',');
 
-        relay.marker = new google.maps.Marker({
-          map: $scope.map,
-          position: new google.maps.LatLng(locParts[0], locParts[1]),
+        relay.marker = new L.Marker([relay.location.latitude, relay.location.longitude],{
           title: relay.url,
         });
 
         var scope = $rootScope.$new(true);
         scope.relay = relay;
 
-        relay.marker.info = new google.maps.InfoWindow({
-          content: $compile($scope.tooltipTemplate)(scope)[0],
+        var icon = new L.Icon({
+          iconSize:     [18, 28], // size of the icon
+          iconAnchor:   [9, 28], // point of the icon which will correspond to marker's location
+          shadowAnchor: [0, 0],  // the same for the shadow
+          popupAnchor:  [0, -27], // popup anchor
+          shadowSize:   [0,0],
+          iconUrl: 'https://cdn.rawgit.com/pointhi/leaflet-color-markers/master/img/marker-icon-red.png',
+          shadowUrl: 'https://cdnjs.cloudflare.com/ajax/libs/leaflet/0.7.7/images/marker-shadow.png',
         });
 
+        relay.marker = new L.marker(new L.latLng(locParts[0], locParts[1]),{icon})
+                            .bindPopup($compile($scope.tooltipTemplate)(scope)[0],{})
+                            .on('mouseover', function (e) {
+                                this.openPopup();
+                            }).on('mouseout', function (e) {
+                                this.closePopup();
+                            }).addTo($scope.map);
+
         relay.showMarker = function() {
-          relay.marker.info.open($scope.map, relay.marker);
+          relay.marker.openPopup();
         }
-
+        
         relay.hideMarker = function() {
-          relay.marker.info.close();
+          relay.marker.closePopup();
         }
+      }
 
-        relay.marker.addListener('mouseover', relay.showMarker);
-        relay.marker.addListener('mouseout', relay.hideMarker);
-
-        $scope.mapBounds.extend(relay.marker.position);
-    }
 
     function addCircleToMap(relay) {
-      relay.marker.circle = new google.maps.Circle({
-        strokeColor: '#FF0000',
-        strokeOpacity: 0.8,
-        strokeWeight: 2,
-        fillColor: '#FF0000',
-        fillOpacity: 0.35,
-        map: $scope.map,
-        center: relay.marker.position,
-        radius: ((relay.status.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000
-      });
-    }
-
-    function getRelayStatus(relay) {
-      // Normal timeout doesn't deal with relays which accept the TCP connection
-      // but don't respond (some firewalls do that), so deal with it this way.
-      var timeoutRequest = $q.defer();
-      var resolveStatus = $q.defer();
-
-
-      $http.get("http://" + relay.uri.hostname + ':' + ((relay.uri.args.statusAddr && relay.uri.args.statusAddr.split(':')[1]) || "22070") + "/status", { timeout: timeoutRequest.promise }).then(function (response) {
-        relay.status = response.data;
-        resolveStatus.resolve();
-        angular.forEach($scope.totals, function(value, key) {
-          if (typeof $scope.totals[key] == 'number') {
-            $scope.totals[key] += response.data[key];
-          } else if (typeof $scope.totals[key] == 'object' && $scope.totals[key] instanceof Array) {
-            angular.forEach($scope.totals[key], function(value, index) {
-              $scope.totals[key][index] += response.data[key][index];
-            });
-          }
-        });
-      }, function() {
-        relay.status = null;
-        resolveStatus.resolve();
-      });
-
-      $timeout(function() {
-        timeoutRequest.resolve();
-      }, 5000);
-
-      return resolveStatus.promise;
+      console.log(relay.location.latitude)
+      L.circle([relay.location.latitude, relay.location.longitude], 
+        {
+          radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000,
+          color: "FF0000",
+          fillColor: "#FF0000",
+          fillOpacity: 0.35,
+        }).addTo($scope.map);
     }
 
     function constructURI(url) {
@@ -385,25 +376,25 @@
 
   <script type="text/template" id="infoTemplate">
     <div>
-      <p><b>{{ relay.uri.hostname }}</b> <span ng-if="relay.status.options['provided-by']">provided by <u>{{ relay.status.options['provided-by'] }}</u></span></p>
-      <div ng-if="relay.status">
-        <span ng-if="relay.status.startTime">Start time: {{ relay.status.startTime | date:"medium" }}</br></span>
-        <span ng-if="relay.status.bytesProxied != undefined">Proxied: {{ relay.status.bytesProxied | bytes }}</br></span>
-        <span ng-if="relay.status.numActiveSessions != undefined">Sessions: {{ relay.status.numActiveSessions }}</br></span>
-        <span ng-if="relay.status.numConnections != undefined">Clients: {{ relay.status.numConnections }}</br></span>
-        <span ng-if="relay.status.options.pools">Pools: {{ relay.status.options.pools.join(', ') }}</br></span>
-        <span ng-if="relay.status.options['global-rate'] != undefined">
-          <span ng-if="relay.status.options['global-rate'] > 0">Global rate limit: {{ relay.status.options['global-rate'] | bytes }}/s</span>
-          <span ng-if="relay.status.options['global-rate'] == 0">Global rate limit: unlimited</span>
+      <p><b>{{ relay.uri.hostname }}</b> <span ng-if="relay.stats.options['provided-by']">provided by <u>{{ relay.stats.options['provided-by'] }}</u></span></p>
+      <div ng-if="relay.stats">
+        <span ng-if="relay.stats.startTime">Start time: {{ relay.stats.startTime | date:"medium" }}</br></span>
+        <span ng-if="relay.stats.bytesProxied != undefined">Proxied: {{ relay.stats.bytesProxied | bytes }}</br></span>
+        <span ng-if="relay.stats.numActiveSessions != undefined">Sessions: {{ relay.stats.numActiveSessions }}</br></span>
+        <span ng-if="relay.stats.numConnections != undefined">Clients: {{ relay.stats.numConnections }}</br></span>
+        <span ng-if="relay.stats.options.pools">Pools: {{ relay.stats.options.pools.join(', ') }}</br></span>
+        <span ng-if="relay.stats.options['global-rate'] != undefined">
+          <span ng-if="relay.stats.options['global-rate'] > 0">Global rate limit: {{ relay.stats.options['global-rate'] | bytes }}/s</span>
+          <span ng-if="relay.stats.options['global-rate'] == 0">Global rate limit: unlimited</span>
           <br/>
         </span>
-        <span ng-if="relay.status.options['per-session-rate'] != undefined">
-          <span ng-if="relay.status.options['per-session-rate'] > 0">Session rate limit: {{ relay.status.options['per-session-rate'] | bytes }}/s</span>
-          <span ng-if="relay.status.options['per-session-rate'] == 0">Session rate limit: unlimited</span>
+        <span ng-if="relay.stats.options['per-session-rate'] != undefined">
+          <span ng-if="relay.stats.options['per-session-rate'] > 0">Session rate limit: {{ relay.stats.options['per-session-rate'] | bytes }}/s</span>
+          <span ng-if="relay.stats.options['per-session-rate'] == 0">Session rate limit: unlimited</span>
           <br/>
         </span>
       </div>
-      <div ng-if="!relay.status">
+      <div ng-if="!relay.stats">
         Data unavailable.
       <div>
     </div>

cmd/strelaypoolsrv/main.go

@@ -1,30 +1,35 @@
 // Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
 
-//go:generate go run ../../script/genassets.go gui >auto/gui.go
-
 package main
 
 import (
-	"bytes"
 	"compress/gzip"
+	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
-	"math/rand"
-	"mime"
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/protocol"
+
 	"github.com/golang/groupcache/lru"
 	"github.com/oschwald/geoip2-golang"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
+	"github.com/syncthing/syncthing/lib/assets"
+	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/relay/client"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -34,12 +39,42 @@ import (
 type location struct {
 	Latitude  float64 `json:"latitude"`
 	Longitude float64 `json:"longitude"`
+	City      string  `json:"city"`
+	Country   string  `json:"country"`
+	Continent string  `json:"continent"`
 }
 
 type relay struct {
-	URL      string   `json:"url"`
-	Location location `json:"location"`
-	uri      *url.URL
+	URL            string   `json:"url"`
+	Location       location `json:"location"`
+	uri            *url.URL
+	Stats          *stats    `json:"stats"`
+	StatsRetrieved time.Time `json:"statsRetrieved"`
+}
+
+type stats struct {
+	StartTime          time.Time `json:"startTime"`
+	UptimeSeconds      int       `json:"uptimeSeconds"`
+	PendingSessionKeys int       `json:"numPendingSessionKeys"`
+	ActiveSessions     int       `json:"numActiveSessions"`
+	Connections        int       `json:"numConnections"`
+	Proxies            int       `json:"numProxies"`
+	BytesProxied       int       `json:"bytesProxied"`
+	GoVersion          string    `json:"goVersion"`
+	GoOS               string    `json:"goOS"`
+	GoArch             string    `json:"goArch"`
+	GoMaxProcs         int       `json:"goMaxProcs"`
+	GoRoutines         int       `json:"goNumRoutine"`
+	Rates              []int64   `json:"kbps10s1m5m15m30m60m"`
+	Options            struct {
+		NetworkTimeout int      `json:"network-timeout"`
+		PintInterval   int      `json:"ping-interval"`
+		MessageTimeout int      `json:"message-timeout"`
+		SessionRate    int      `json:"per-session-rate"`
+		GlobalRate     int      `json:"global-rate"`
+		Pools          []string `json:"pools"`
+		ProvidedBy     string   `json:"provided-by"`
+	} `json:"options"`
 }
 
 func (r relay) String() string {
@@ -47,9 +82,9 @@ func (r relay) String() string {
 }
 
 type request struct {
-	relay  relay
-	uri    *url.URL
-	result chan result
+	relay      *relay
+	result     chan result
+	queueTimer *prometheus.Timer
 }
 
 type result struct {
@@ -58,56 +93,72 @@ type result struct {
 }
 
 var (
-	testCert       tls.Certificate
-	listen         = ":80"
-	dir            string
-	evictionTime   = time.Hour
-	debug          bool
-	getLRUSize     = 10 << 10
-	getLimitBurst  = 10
-	getLimitAvg    = 1
-	postLRUSize    = 1 << 10
-	postLimitBurst = 2
-	postLimitAvg   = 1
-	getLimit       time.Duration
-	postLimit      time.Duration
-	permRelaysFile string
-	ipHeader       string
-	geoipPath      string
-	proto          string
+	testCert          tls.Certificate
+	knownRelaysFile   = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
+	listen            = ":80"
+	dir               string
+	evictionTime      = time.Hour
+	debug             bool
+	getLRUSize        = 10 << 10
+	getLimitBurst     = 10
+	getLimitAvg       = 2
+	postLRUSize       = 1 << 10
+	postLimitBurst    = 2
+	postLimitAvg      = 2
+	getLimit          time.Duration
+	postLimit         time.Duration
+	permRelaysFile    string
+	ipHeader          string
+	geoipPath         string
+	proto             string
+	statsRefresh      = time.Minute / 2
+	requestQueueLen   = 10
+	requestProcessors = 1
 
-	getMut      = sync.NewRWMutex()
+	getMut      = sync.NewMutex()
 	getLRUCache *lru.Cache
 
-	postMut      = sync.NewRWMutex()
+	postMut      = sync.NewMutex()
 	postLRUCache *lru.Cache
 
-	requests = make(chan request, 10)
+	requests chan request
 
 	mut             = sync.NewRWMutex()
-	knownRelays     = make([]relay, 0)
-	permanentRelays = make([]relay, 0)
+	knownRelays     = make([]*relay, 0)
+	permanentRelays = make([]*relay, 0)
 	evictionTimers  = make(map[string]*time.Timer)
 )
 
+const (
+	httpStatusEnhanceYourCalm = 429
+)
+
 func main() {
+	log.SetOutput(os.Stdout)
+	log.SetFlags(log.Lshortfile)
+
 	flag.StringVar(&listen, "listen", listen, "Listen address")
 	flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
 	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
 	flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
 	flag.IntVar(&getLRUSize, "get-limit-cache", getLRUSize, "Get request limiter cache size")
-	flag.IntVar(&getLimitAvg, "get-limit-avg", 2, "Allowed average get request rate, per 10 s")
+	flag.IntVar(&getLimitAvg, "get-limit-avg", getLimitAvg, "Allowed average get request rate, per 10 s")
 	flag.IntVar(&getLimitBurst, "get-limit-burst", getLimitBurst, "Allowed burst get requests")
 	flag.IntVar(&postLRUSize, "post-limit-cache", postLRUSize, "Post request limiter cache size")
-	flag.IntVar(&postLimitAvg, "post-limit-avg", 2, "Allowed average post request rate, per minute")
+	flag.IntVar(&postLimitAvg, "post-limit-avg", postLimitAvg, "Allowed average post request rate, per minute")
 	flag.IntVar(&postLimitBurst, "post-limit-burst", postLimitBurst, "Allowed burst post requests")
 	flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
 	flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
 	flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
 	flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
+	flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
+	flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
+	flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
 
 	flag.Parse()
 
+	requests = make(chan request, requestQueueLen)
+
 	getLimit = 10 * time.Second / time.Duration(getLimitAvg)
 	postLimit = time.Minute / time.Duration(postLimitAvg)
 
@@ -118,12 +169,32 @@ func main() {
 	var err error
 
 	if permRelaysFile != "" {
-		loadPermanentRelays(permRelaysFile)
+		permanentRelays = loadRelays(permRelaysFile)
 	}
 
 	testCert = createTestCertificate()
 
-	go requestProcessor()
+	for i := 0; i < requestProcessors; i++ {
+		go requestProcessor()
+	}
+
+	// Load relays from cache in the background.
+	// Load them in a serial fashion to make sure any genuine requests
+	// are not dropped.
+	go func() {
+		for _, relay := range loadRelays(knownRelaysFile) {
+			resultChan := make(chan result)
+			requests <- request{relay, resultChan, nil}
+			result := <-resultChan
+			if result.err != nil {
+				relayTestsTotal.WithLabelValues("failed").Inc()
+			} else {
+				relayTestsTotal.WithLabelValues("success").Inc()
+			}
+		}
+		// Run the the stats refresher once the relays are loaded.
+		statsRefresher(statsRefresh)
+	}()
 
 	if dir != "" {
 		if debug {
@@ -139,6 +210,7 @@ func main() {
 		tlsCfg := &tls.Config{
 			Certificates: []tls.Certificate{cert},
 			MinVersion:   tls.VersionTLS10, // No SSLv3
+			ClientAuth:   tls.RequestClientCert,
 			CipherSuites: []uint16{
 				// No RC4
 				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
@@ -169,6 +241,7 @@ func main() {
 	handler := http.NewServeMux()
 	handler.HandleFunc("/", handleAssets)
 	handler.HandleFunc("/endpoint", handleRequest)
+	handler.HandleFunc("/metrics", handleMetrics)
 
 	srv := http.Server{
 		Handler:     handler,
@@ -181,66 +254,42 @@ func main() {
 	}
 }
 
+func handleMetrics(w http.ResponseWriter, r *http.Request) {
+	timer := prometheus.NewTimer(metricsRequestsSeconds)
+	// Acquire the mutex just to make sure we're not caught mid-way stats collection
+	mut.RLock()
+	promhttp.Handler().ServeHTTP(w, r)
+	mut.RUnlock()
+	timer.ObserveDuration()
+}
+
 func handleAssets(w http.ResponseWriter, r *http.Request) {
-	assets := auto.Assets()
+	w.Header().Set("Cache-Control", "no-cache, must-revalidate")
+
 	path := r.URL.Path[1:]
 	if path == "" {
 		path = "index.html"
 	}
 
-	bs, ok := assets[path]
+	as, ok := auto.Assets()[path]
 	if !ok {
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
 
-	mtype := mimeTypeForFile(path)
-	if len(mtype) != 0 {
-		w.Header().Set("Content-Type", mtype)
-	}
-
-	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
-		w.Header().Set("Content-Encoding", "gzip")
-	} else {
-		// ungzip if browser not send gzip accepted header
-		var gr *gzip.Reader
-		gr, _ = gzip.NewReader(bytes.NewReader(bs))
-		bs, _ = ioutil.ReadAll(gr)
-		gr.Close()
-	}
-	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs)))
-
-	w.Write(bs)
-}
-
-func mimeTypeForFile(file string) string {
-	// We use a built in table of the common types since the system
-	// TypeByExtension might be unreliable. But if we don't know, we delegate
-	// to the system.
-	ext := filepath.Ext(file)
-	switch ext {
-	case ".htm", ".html":
-		return "text/html"
-	case ".css":
-		return "text/css"
-	case ".js":
-		return "application/javascript"
-	case ".json":
-		return "application/json"
-	case ".png":
-		return "image/png"
-	case ".ttf":
-		return "application/x-font-ttf"
-	case ".woff":
-		return "application/x-font-woff"
-	case ".svg":
-		return "image/svg+xml"
-	default:
-		return mime.TypeByExtension(ext)
-	}
+	assets.Serve(w, r, as)
 }
 
 func handleRequest(w http.ResponseWriter, r *http.Request) {
+	timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
+
+	w = NewLoggingResponseWriter(w)
+	defer func() {
+		timer.ObserveDuration()
+		lw := w.(*loggingResponseWriter)
+		apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
+	}()
+
 	if ipHeader != "" {
 		r.RemoteAddr = r.Header.Get(ipHeader)
 	}
@@ -248,13 +297,13 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 	switch r.Method {
 	case "GET":
 		if limit(r.RemoteAddr, getLRUCache, getMut, getLimit, getLimitBurst) {
-			w.WriteHeader(429)
+			w.WriteHeader(httpStatusEnhanceYourCalm)
 			return
 		}
 		handleGetRequest(w, r)
 	case "POST":
 		if limit(r.RemoteAddr, postLRUCache, postMut, postLimit, postLimitBurst) {
-			w.WriteHeader(429)
+			w.WriteHeader(httpStatusEnhanceYourCalm)
 			return
 		}
 		handlePostRequest(w, r)
@@ -266,24 +315,38 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func handleGetRequest(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+func handleGetRequest(rw http.ResponseWriter, r *http.Request) {
+	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
+
 	mut.RLock()
-	relays := append(permanentRelays, knownRelays...)
+	relays := make([]*relay, len(permanentRelays)+len(knownRelays))
+	n := copy(relays, permanentRelays)
+	copy(relays[n:], knownRelays)
 	mut.RUnlock()
 
 	// Shuffle
-	for i := range relays {
-		j := rand.Intn(i + 1)
-		relays[i], relays[j] = relays[j], relays[i]
+	rand.Shuffle(relays)
+
+	w := io.Writer(rw)
+	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+		rw.Header().Set("Content-Encoding", "gzip")
+		gw := gzip.NewWriter(rw)
+		defer gw.Close()
+		w = gw
 	}
 
-	json.NewEncoder(w).Encode(map[string][]relay{
+	_ = json.NewEncoder(w).Encode(map[string][]*relay{
 		"relays": relays,
 	})
 }
 
 func handlePostRequest(w http.ResponseWriter, r *http.Request) {
+	var relayCert *x509.Certificate
+	if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
+		relayCert = r.TLS.PeerCertificates[0]
+		log.Printf("Got TLS cert from relay server")
+	}
+
 	var newRelay relay
 	err := json.NewDecoder(r.Body).Decode(&newRelay)
 	r.Body.Close()
@@ -292,7 +355,7 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		if debug {
 			log.Println("Failed to parse payload")
 		}
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
@@ -301,27 +364,38 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		if debug {
 			log.Println("Failed to parse URI", newRelay.URL)
 		}
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
+	// Canonicalize the URL. In particular, parse and re-encode the query
+	// string so that it's guaranteed to be valid.
+	uri.RawQuery = uri.Query().Encode()
+	newRelay.URL = uri.String()
+
+	if relayCert != nil {
+		advertisedId := uri.Query().Get("id")
+		idFromCert := protocol.NewDeviceID(relayCert.Raw).String()
+		if advertisedId != idFromCert {
+			log.Println("Warning: Relay server requested to join with an ID different from the join request, rejecting")
+			http.Error(w, "mismatched advertised id and join request cert", http.StatusBadRequest)
+			return
+		}
+	}
+
 	host, port, err := net.SplitHostPort(uri.Host)
 	if err != nil {
 		if debug {
 			log.Println("Failed to split URI", newRelay.URL)
 		}
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
 	// Get the IP address of the client
-	rhost, _, err := net.SplitHostPort(r.RemoteAddr)
-	if err != nil {
-		if debug {
-			log.Println("Failed to split remote address", r.RemoteAddr)
-		}
-		http.Error(w, err.Error(), 500)
-		return
+	rhost := r.RemoteAddr
+	if host, _, err := net.SplitHostPort(rhost); err == nil {
+		rhost = host
 	}
 
 	ip := net.ParseIP(host)
@@ -329,22 +403,22 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 	if ip == nil || ip.IsUnspecified() {
 		uri.Host = net.JoinHostPort(rhost, port)
 		newRelay.URL = uri.String()
-	} else if host != rhost {
+	} else if host != rhost && relayCert == nil {
 		if debug {
 			log.Println("IP address advertised does not match client IP address", r.RemoteAddr, uri)
 		}
-		http.Error(w, "IP address does not match client IP", http.StatusUnauthorized)
+		http.Error(w, fmt.Sprintf("IP advertised %s does not match client IP %s", host, rhost), http.StatusUnauthorized)
 		return
 	}
+
 	newRelay.uri = uri
-	newRelay.Location = getLocation(uri.Host)
 
 	for _, current := range permanentRelays {
 		if current.uri.Host == newRelay.uri.Host {
 			if debug {
 				log.Println("Asked to add a relay", newRelay, "which exists in permanent list")
 			}
-			http.Error(w, "Invalid request", 500)
+			http.Error(w, "Invalid request", http.StatusBadRequest)
 			return
 		}
 	}
@@ -352,78 +426,105 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 	reschan := make(chan result)
 
 	select {
-	case requests <- request{newRelay, uri, reschan}:
+	case requests <- request{&newRelay, reschan, prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("queue"))}:
 		result := <-reschan
 		if result.err != nil {
-			http.Error(w, result.err.Error(), 500)
+			relayTestsTotal.WithLabelValues("failed").Inc()
+			http.Error(w, result.err.Error(), http.StatusBadRequest)
 			return
 		}
+		relayTestsTotal.WithLabelValues("success").Inc()
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		json.NewEncoder(w).Encode(map[string]time.Duration{
 			"evictionIn": result.eviction,
 		})
 
 	default:
+		relayTestsTotal.WithLabelValues("dropped").Inc()
 		if debug {
 			log.Println("Dropping request")
 		}
-		w.WriteHeader(429)
+		w.WriteHeader(httpStatusEnhanceYourCalm)
 	}
 }
 
 func requestProcessor() {
 	for request := range requests {
-		if debug {
-			log.Println("Request for", request.relay)
-		}
-		if !client.TestRelay(request.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
-			if debug {
-				log.Println("Test for relay", request.relay, "failed")
-			}
-			request.result <- result{fmt.Errorf("test failed"), 0}
-			continue
+		if request.queueTimer != nil {
+			request.queueTimer.ObserveDuration()
 		}
 
-		mut.Lock()
-		timer, ok := evictionTimers[request.relay.uri.Host]
-		if ok {
-			if debug {
-				log.Println("Stopping existing timer for", request.relay)
-			}
-			timer.Stop()
-		}
-
-		for i, current := range knownRelays {
-			if current.uri.Host == request.relay.uri.Host {
-				if debug {
-					log.Println("Relay", request.relay, "already exists")
-				}
-
-				// Evict the old entry anyway, as configuration might have changed.
-				last := len(knownRelays) - 1
-				knownRelays[i] = knownRelays[last]
-				knownRelays = knownRelays[:last]
-
-				goto found
-			}
-		}
-
-		if debug {
-			log.Println("Adding new relay", request.relay)
-		}
-
-	found:
-
-		knownRelays = append(knownRelays, request.relay)
-
-		evictionTimers[request.relay.uri.Host] = time.AfterFunc(evictionTime, evict(request.relay))
-		mut.Unlock()
-		request.result <- result{nil, evictionTime}
+		timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test"))
+		handleRelayTest(request)
+		timer.ObserveDuration()
 	}
-
 }
 
-func evict(relay relay) func() {
+func handleRelayTest(request request) {
+	if debug {
+		log.Println("Request for", request.relay)
+	}
+	if err := client.TestRelay(context.TODO(), request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3); err != nil {
+		if debug {
+			log.Println("Test for relay", request.relay, "failed:", err)
+		}
+		request.result <- result{err, 0}
+		return
+	}
+
+	stats := fetchStats(request.relay)
+	location := getLocation(request.relay.uri.Host)
+
+	mut.Lock()
+	if stats != nil {
+		updateMetrics(request.relay.uri.Host, *stats, location)
+	}
+	request.relay.Stats = stats
+	request.relay.StatsRetrieved = time.Now().Truncate(time.Second)
+	request.relay.Location = location
+
+	timer, ok := evictionTimers[request.relay.uri.Host]
+	if ok {
+		if debug {
+			log.Println("Stopping existing timer for", request.relay)
+		}
+		timer.Stop()
+	}
+
+	for i, current := range knownRelays {
+		if current.uri.Host == request.relay.uri.Host {
+			if debug {
+				log.Println("Relay", request.relay, "already exists")
+			}
+
+			// Evict the old entry anyway, as configuration might have changed.
+			last := len(knownRelays) - 1
+			knownRelays[i] = knownRelays[last]
+			knownRelays = knownRelays[:last]
+
+			goto found
+		}
+	}
+
+	if debug {
+		log.Println("Adding new relay", request.relay)
+	}
+
+found:
+
+	knownRelays = append(knownRelays, request.relay)
+	evictionTimers[request.relay.uri.Host] = time.AfterFunc(evictionTime, evict(request.relay))
+
+	mut.Unlock()
+
+	if err := saveRelays(knownRelaysFile, knownRelays); err != nil {
+		log.Println("Failed to write known relays: " + err.Error())
+	}
+
+	request.result <- result{nil, evictionTime}
+}
+
+func evict(relay *relay) func() {
 	return func() {
 		mut.Lock()
 		defer mut.Unlock()
@@ -438,74 +539,80 @@ func evict(relay relay) func() {
 				last := len(knownRelays) - 1
 				knownRelays[i] = knownRelays[last]
 				knownRelays = knownRelays[:last]
+				deleteMetrics(current.uri.Host)
 			}
 		}
 		delete(evictionTimers, relay.uri.Host)
 	}
 }
 
-func limit(addr string, cache *lru.Cache, lock sync.RWMutex, intv time.Duration, burst int) bool {
-	host, _, err := net.SplitHostPort(addr)
-	if err != nil {
-		return false
+func limit(addr string, cache *lru.Cache, lock sync.Mutex, intv time.Duration, burst int) bool {
+	if host, _, err := net.SplitHostPort(addr); err == nil {
+		addr = host
 	}
 
-	lock.RLock()
-	bkt, ok := cache.Get(host)
-	lock.RUnlock()
-	if ok {
-		bkt := bkt.(*rate.Limiter)
-		if !bkt.Allow() {
-			// Rate limit
-			return true
-		}
-	} else {
-		lock.Lock()
-		cache.Add(host, rate.NewLimiter(rate.Every(intv), burst))
-		lock.Unlock()
+	lock.Lock()
+	v, _ := cache.Get(addr)
+	bkt, ok := v.(*rate.Limiter)
+	if !ok {
+		bkt = rate.NewLimiter(rate.Every(intv), burst)
+		cache.Add(addr, bkt)
 	}
-	return false
+	lock.Unlock()
+
+	return !bkt.Allow()
 }
 
-func loadPermanentRelays(file string) {
-	content, err := ioutil.ReadFile(file)
+func loadRelays(file string) []*relay {
+	content, err := os.ReadFile(file)
 	if err != nil {
-		log.Fatal(err)
+		log.Println("Failed to load relays: " + err.Error())
+		return nil
 	}
 
+	var relays []*relay
 	for _, line := range strings.Split(string(content), "\n") {
-		if len(line) == 0 {
+		if line == "" {
 			continue
 		}
 
 		uri, err := url.Parse(line)
 		if err != nil {
 			if debug {
-				log.Println("Skipping permanent relay", line, "due to parse error", err)
+				log.Println("Skipping relay", line, "due to parse error", err)
 			}
 			continue
 
 		}
 
-		permanentRelays = append(permanentRelays, relay{
+		relays = append(relays, &relay{
 			URL:      line,
 			Location: getLocation(uri.Host),
 			uri:      uri,
 		})
 		if debug {
-			log.Println("Adding permanent relay", line)
+			log.Println("Adding relay", line)
 		}
 	}
+	return relays
+}
+
+func saveRelays(file string, relays []*relay) error {
+	var content string
+	for _, relay := range relays {
+		content += relay.uri.String() + "\n"
+	}
+	return os.WriteFile(file, []byte(content), 0777)
 }
 
 func createTestCertificate() tls.Certificate {
-	tmpDir, err := ioutil.TempDir("", "relaypoolsrv")
+	tmpDir, err := os.MkdirTemp("", "relaypoolsrv")
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
-	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 3072)
+	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 20*365)
 	if err != nil {
 		log.Fatalln("Failed to create test X509 key pair:", err)
 	}
@@ -514,6 +621,8 @@ func createTestCertificate() tls.Certificate {
 }
 
 func getLocation(host string) location {
+	timer := prometheus.NewTimer(locationLookupSeconds)
+	defer timer.ObserveDuration()
 	db, err := geoip2.Open(geoipPath)
 	if err != nil {
 		return location{}
@@ -531,7 +640,24 @@ func getLocation(host string) location {
 	}
 
 	return location{
-		Latitude:  city.Location.Latitude,
 		Longitude: city.Location.Longitude,
+		Latitude:  city.Location.Latitude,
+		City:      city.City.Names["en"],
+		Country:   city.Country.IsoCode,
+		Continent: city.Continent.Code,
 	}
 }
+
+type loggingResponseWriter struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
+	return &loggingResponseWriter{w, http.StatusOK}
+}
+
+func (lrw *loggingResponseWriter) WriteHeader(code int) {
+	lrw.statusCode = code
+	lrw.ResponseWriter.WriteHeader(code)
+}

cmd/strelaypoolsrv/main_test.go

@@ -0,0 +1,94 @@
+// Copyright © 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"sync"
+	"testing"
+)
+
+func init() {
+	for i := 0; i < 10; i++ {
+		u := fmt.Sprintf("permanent%d", i)
+		permanentRelays = append(permanentRelays, &relay{URL: u})
+	}
+
+	knownRelays = []*relay{
+		{URL: "known1"},
+		{URL: "known2"},
+		{URL: "known3"},
+	}
+
+	mut = new(sync.RWMutex)
+}
+
+// Regression test: handleGetRequest should not modify permanentRelays.
+func TestHandleGetRequest(t *testing.T) {
+	needcap := len(permanentRelays) + len(knownRelays)
+	if needcap > cap(permanentRelays) {
+		t.Fatalf("test setup failed: need cap(permanentRelays) >= %d, have %d",
+			needcap, cap(permanentRelays))
+	}
+
+	w := httptest.NewRecorder()
+	w.Body = new(bytes.Buffer)
+	handleGetRequest(w, httptest.NewRequest("GET", "/", nil))
+
+	result := make(map[string][]*relay)
+	err := json.NewDecoder(w.Body).Decode(&result)
+	if err != nil {
+		t.Fatalf("invalid JSON: %v", err)
+	}
+
+	relays := result["relays"]
+	expect, actual := len(knownRelays)+len(permanentRelays), len(relays)
+	if actual != expect {
+		t.Errorf("expected %d relays, got %d", expect, actual)
+	}
+
+	// Check for changes in permanentRelays.
+	for i, r := range permanentRelays {
+		switch {
+		case !strings.HasPrefix(r.URL, "permanent"):
+			t.Errorf("relay %q among permanent relays", r.URL)
+		case r.URL != fmt.Sprintf("permanent%d", i):
+			t.Error("order of permanent relays changed")
+		}
+	}
+}
+
+func TestCanonicalizeQueryValues(t *testing.T) {
+	// This just demonstrates and validates the uri.Parse/String stuff in
+	// regards to query strings.
+
+	in := "http://example.com/?some weird= query^value"
+	exp := "http://example.com/?some+weird=+query%5Evalue"
+
+	uri, err := url.Parse(in)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	str := uri.String()
+	if str != in {
+		// Just re-encoding the URL doesn't sanitize the query string.
+		t.Errorf("expected %q, got %q", in, str)
+	}
+
+	uri.RawQuery = uri.Query().Encode()
+	str = uri.String()
+	if str != exp {
+		// The query string is now in correct format.
+		t.Errorf("expected %q, got %q", exp, str)
+	}
+}

cmd/strelaypoolsrv/stats.go

@@ -0,0 +1,260 @@
+// Copyright (C) 2018 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"encoding/json"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+func init() {
+	processCollectorOpts := collectors.ProcessCollectorOpts{
+		Namespace: "syncthing_relaypoolsrv",
+		PidFn: func() (int, error) {
+			return os.Getpid(), nil
+		},
+	}
+
+	prometheus.MustRegister(
+		collectors.NewProcessCollector(processCollectorOpts),
+	)
+}
+
+var (
+	statusClient = http.Client{
+		Timeout: 5 * time.Second,
+	}
+
+	apiRequestsTotal   = makeCounter("api_requests_total", "Number of API requests.", "type", "result")
+	apiRequestsSeconds = makeSummary("api_requests_seconds", "Latency of API requests.", "type")
+
+	relayTestsTotal         = makeCounter("tests_total", "Number of relay tests.", "result")
+	relayTestActionsSeconds = makeSummary("test_actions_seconds", "Latency of relay test actions.", "type")
+
+	locationLookupSeconds = makeSummary("location_lookup_seconds", "Latency of location lookups.").WithLabelValues()
+
+	metricsRequestsSeconds = makeSummary("metrics_requests_seconds", "Latency of metric requests.").WithLabelValues()
+	scrapeSeconds          = makeSummary("relay_scrape_seconds", "Latency of metric scrapes from remote relays.", "result")
+
+	relayUptime             = makeGauge("relay_uptime", "Uptime of relay", "relay")
+	relayPendingSessionKeys = makeGauge("relay_pending_session_keys", "Number of pending session keys (two keys per session, one per each side of the connection)", "relay")
+	relayActiveSessions     = makeGauge("relay_active_sessions", "Number of sessions that are happening, a session contains two parties", "relay")
+	relayConnections        = makeGauge("relay_connections", "Number of devices connected to the relay", "relay")
+	relayProxies            = makeGauge("relay_proxies", "Number of active proxy routines sending data between peers (two proxies per session, one for each way)", "relay")
+	relayBytesProxied       = makeGauge("relay_bytes_proxied", "Number of bytes proxied by the relay", "relay")
+	relayGoRoutines         = makeGauge("relay_go_routines", "Number of Go routines in the process", "relay")
+	relaySessionRate        = makeGauge("relay_session_rate", "Rate applied per session", "relay")
+	relayGlobalRate         = makeGauge("relay_global_rate", "Global rate applied on the whole relay", "relay")
+	relayBuildInfo          = makeGauge("relay_build_info", "Build information about a relay", "relay", "go_version", "go_os", "go_arch")
+	relayLocationInfo       = makeGauge("relay_location_info", "Location information about a relay", "relay", "city", "country", "continent")
+
+	lastStats = make(map[string]stats)
+)
+
+func makeGauge(name string, help string, labels ...string) *prometheus.GaugeVec {
+	gauge := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "syncthing",
+			Subsystem: "relaypoolsrv",
+			Name:      name,
+			Help:      help,
+		},
+		labels,
+	)
+	prometheus.MustRegister(gauge)
+	return gauge
+}
+
+func makeSummary(name string, help string, labels ...string) *prometheus.SummaryVec {
+	summary := prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Namespace:  "syncthing",
+			Subsystem:  "relaypoolsrv",
+			Name:       name,
+			Help:       help,
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		labels,
+	)
+	prometheus.MustRegister(summary)
+	return summary
+}
+
+func makeCounter(name string, help string, labels ...string) *prometheus.CounterVec {
+	counter := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "syncthing",
+			Subsystem: "relaypoolsrv",
+			Name:      name,
+			Help:      help,
+		},
+		labels,
+	)
+	prometheus.MustRegister(counter)
+	return counter
+}
+
+func statsRefresher(interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	for range ticker.C {
+		refreshStats()
+	}
+}
+
+type statsFetchResult struct {
+	relay *relay
+	stats *stats
+}
+
+func refreshStats() {
+	mut.RLock()
+	relays := append(permanentRelays, knownRelays...)
+	mut.RUnlock()
+
+	now := time.Now()
+	wg := sync.NewWaitGroup()
+
+	results := make(chan statsFetchResult, len(relays))
+	for _, rel := range relays {
+		wg.Add(1)
+		go func(rel *relay) {
+			t0 := time.Now()
+			stats := fetchStats(rel)
+			duration := time.Since(t0).Seconds()
+			result := "success"
+			if stats == nil {
+				result = "failed"
+			}
+			scrapeSeconds.WithLabelValues(result).Observe(duration)
+
+			results <- statsFetchResult{
+				relay: rel,
+				stats: fetchStats(rel),
+			}
+			wg.Done()
+		}(rel)
+	}
+
+	wg.Wait()
+	close(results)
+
+	mut.Lock()
+	relayBuildInfo.Reset()
+	relayLocationInfo.Reset()
+	for result := range results {
+		result.relay.StatsRetrieved = now
+		result.relay.Stats = result.stats
+		if result.stats == nil {
+			deleteMetrics(result.relay.uri.Host)
+		} else {
+			updateMetrics(result.relay.uri.Host, *result.stats, result.relay.Location)
+		}
+	}
+	mut.Unlock()
+}
+
+func fetchStats(relay *relay) *stats {
+	statusAddr := relay.uri.Query().Get("statusAddr")
+	if statusAddr == "" {
+		statusAddr = ":22070"
+	}
+
+	statusHost, statusPort, err := net.SplitHostPort(statusAddr)
+	if err != nil {
+		return nil
+	}
+
+	if statusHost == "" {
+		if host, _, err := net.SplitHostPort(relay.uri.Host); err != nil {
+			return nil
+		} else {
+			statusHost = host
+		}
+	}
+
+	url := "http://" + net.JoinHostPort(statusHost, statusPort) + "/status"
+
+	response, err := statusClient.Get(url)
+	if err != nil {
+		return nil
+	}
+
+	var stats stats
+
+	if json.NewDecoder(response.Body).Decode(&stats); err != nil {
+		return nil
+	}
+	return &stats
+}
+
+func updateMetrics(host string, stats stats, location location) {
+	if stats.GoVersion != "" || stats.GoOS != "" || stats.GoArch != "" {
+		relayBuildInfo.WithLabelValues(host, stats.GoVersion, stats.GoOS, stats.GoArch).Add(1)
+	}
+	if location.City != "" || location.Country != "" || location.Continent != "" {
+		relayLocationInfo.WithLabelValues(host, location.City, location.Country, location.Continent).Add(1)
+	}
+
+	if lastStat, ok := lastStats[host]; ok {
+		stats = mergeStats(stats, lastStat)
+	}
+
+	relayUptime.WithLabelValues(host).Set(float64(stats.UptimeSeconds))
+	relayPendingSessionKeys.WithLabelValues(host).Set(float64(stats.PendingSessionKeys))
+	relayActiveSessions.WithLabelValues(host).Set(float64(stats.ActiveSessions))
+	relayConnections.WithLabelValues(host).Set(float64(stats.Connections))
+	relayProxies.WithLabelValues(host).Set(float64(stats.Proxies))
+	relayBytesProxied.WithLabelValues(host).Set(float64(stats.BytesProxied))
+	relayGoRoutines.WithLabelValues(host).Set(float64(stats.GoRoutines))
+	relaySessionRate.WithLabelValues(host).Set(float64(stats.Options.SessionRate))
+	relayGlobalRate.WithLabelValues(host).Set(float64(stats.Options.GlobalRate))
+	lastStats[host] = stats
+}
+
+func deleteMetrics(host string) {
+	relayUptime.DeleteLabelValues(host)
+	relayPendingSessionKeys.DeleteLabelValues(host)
+	relayActiveSessions.DeleteLabelValues(host)
+	relayConnections.DeleteLabelValues(host)
+	relayProxies.DeleteLabelValues(host)
+	relayBytesProxied.DeleteLabelValues(host)
+	relayGoRoutines.DeleteLabelValues(host)
+	relaySessionRate.DeleteLabelValues(host)
+	relayGlobalRate.DeleteLabelValues(host)
+	delete(lastStats, host)
+}
+
+// Due to some unexplainable behaviour, some of the numbers sometimes travel slightly backwards (by less than 1%)
+// This happens between scrapes, which is 30s, so this can't be a race.
+// This causes prometheus to assume a "rate reset", hence causes phenomenal spikes.
+// One of the number that moves backwards is BytesProxied, which atomically increments a counter with numeric value
+// returned by net.Conn.Read(). I don't think that can return a negative value, so I have no idea what's going on.
+func mergeStats(new stats, old stats) stats {
+	new.UptimeSeconds = mergeValue(new.UptimeSeconds, old.UptimeSeconds)
+	new.PendingSessionKeys = mergeValue(new.PendingSessionKeys, old.PendingSessionKeys)
+	new.ActiveSessions = mergeValue(new.ActiveSessions, old.ActiveSessions)
+	new.Connections = mergeValue(new.Connections, old.Connections)
+	new.Proxies = mergeValue(new.Proxies, old.Proxies)
+	new.BytesProxied = mergeValue(new.BytesProxied, old.BytesProxied)
+	new.GoRoutines = mergeValue(new.GoRoutines, old.GoRoutines)
+	new.Options.SessionRate = mergeValue(new.Options.SessionRate, old.Options.SessionRate)
+	new.Options.GlobalRate = mergeValue(new.Options.GlobalRate, old.Options.GlobalRate)
+	return new
+}
+
+func mergeValue(new, old int) int {
+	if new >= old {
+		return new // normal increase
+	}
+	if float64(new) > 0.99*float64(old) {
+		return old // slight backward movement
+	}
+	return new // reset (relay restart)
+}

cmd/strelaypoolsrv/stats_test.go

@@ -0,0 +1,21 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"testing"
+)
+
+func TestMerge(t *testing.T) {
+	if mergeValue(1001, 1000) != 1001 {
+		t.Error("the computer says no")
+	}
+
+	if mergeValue(999, 1000) != 1000 {
+		t.Error("the computer says no")
+	}
+
+	if mergeValue(1, 1000) != 1 {
+		t.Error("the computer says no")
+	}
+}

cmd/strelaysrv/README.md

@@ -121,11 +121,11 @@ Relay related libraries used by this repo
 ----
 ##### Relay protocol definition.
 
-[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/protocol)
+[Available here](https://github.com/syncthing/syncthing/tree/main/lib/relay/protocol)
 
 
 ##### Relay client
 
 Only used by the testutil.
 
-[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/client)
+[Available here](https://github.com/syncthing/syncthing/tree/main/lib/relay/client)

cmd/strelaysrv/etc/firewall-ufw/strelaysrv

@@ -0,0 +1,9 @@
+[strelaysrv]
+title=Syncthing relay server
+description=Proxies traffic of syncthing client behind firewalls
+ports=22067/tcp
+
+[strelaysrv-metrics]
+title=Syncthing relay metrics
+description=Provides metrics about the syncthing relay server
+ports=22070/tcp

cmd/strelaysrv/etc/linux-systemd/default

@@ -0,0 +1,5 @@
+# Default settings for syncthing-relaysrv (strelaysrv).
+NAT=true
+
+## Add Options here:
+RELAYSRV_OPTS=