gui, man, authors: Update docs, translations, and contributors

In the original fix in #8563 I simply forgot this. Which meant #8556
wasn't actually fixed, as the trialer size would have been 0 (default),
and thus we would have still sent the inflated size to encrypted peers.

.codecov.yml

@@ -1,5 +1,20 @@
+comment: false
+
 coverage:
   range: "40...100"
+  precision: 1
+  status:
+    patch:
+      default:
+        informational: true
+    project:
+      default:
+        informational: true
+
+github_checks:
+  annotations: false
 
 ignore:
   - "**.pb.go"
+  - "**_mocked.go"
+  - "**/mocks/*"

.deepsource.toml

@@ -0,0 +1,12 @@
+version = 1
+
+exclude_patterns = ["**/*.pb.go"]
+test_patterns = ["**/*_test.go"]
+
+[[analyzers]]
+name = "go"
+enabled = true
+
+  [analyzers.meta]
+  import_paths = ["github.com/syncthing/syncthing"]
+  build_tags = ["noassets"]

.github/FUNDING.yml

@@ -0,0 +1,11 @@
+github: syncthing
+custom: "https://syncthing.net/donations/"
+
+# patreon: # Replace with a single Patreon username
+# open_collective: # Replace with a single Open Collective username
+# ko_fi: # Replace with a single Ko-fi username
+# tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+# community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+# liberapay: # Replace with a single Liberapay username
+# issuehunt: # Replace with a single IssueHunt username
+# otechie: # Replace with a single Otechie username

.github/ISSUE_TEMPLATE/config.yml

@@ -1,6 +1,6 @@
 blank_issues_enabled: false
 contact_links:
-  - name: Ask a question
+  - name: I need help / I have a question
     url: https://forum.syncthing.net/
     about: Ask questions, get support, and discuss with other community members.
   - name: Android issues

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 10
+
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 10

.github/workflows/build-syncthing.yaml

@@ -0,0 +1,382 @@
+name: Build Syncthing
+
+on:
+  pull_request:
+  push:
+
+env:
+  # The go version to use for builds.
+  GO_VERSION: "1.19.6"
+
+  # Optimize compatibility on the slow archictures.
+  GO386: softfloat
+  GOARM: "5"
+  GOMIPS: softfloat
+
+  # Avoid hilarious amounts of obscuring log output when running tests.
+  LOGGER_DISCARD: "1"
+
+  # Our build metadata
+  BUILD_USER: builder
+  BUILD_HOST: github.syncthing.net
+
+# A note on actions and third party code... The actions under actions/ (like
+# `uses: actions/checkout`) are maintained by GitHub, and we need to trust
+# GitHub to maintain their code and infrastructure or we're in deep shit in
+# general. The same doesn't necessarily apply to other actions authors, so
+# some care needs to be taken when adding steps, especially in the paths
+# that lead up to code being packaged and signed.
+
+jobs:
+
+  #
+  # Tests for all platforms. Runs a matrix build on Windows, Linux and Mac,
+  # with the list of expected supported Go versions (current, previous).
+  #
+
+  build-test:
+    name: Build and test
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: ["windows-latest", "ubuntu-latest", "macos-latest"]
+        # The oldest version in this list should match what we have in our go.mod.
+        # Variables don't seem to be supported here, or we could have done something nice.
+        go: ["1.19"] # Skip Go 1.20 for now, https://github.com/syncthing/syncthing/issues/8799
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - name: Set git to use LF
+        if: matrix.runner == 'windows-latest'
+        # Without this, the Windows checkout will happen with CRLF line
+        # endings, which is fine for the source code but messes up tests
+        # that depend on data on disk being as expected. Ideally, those
+        # tests should be fixed, but not today.
+        run: |
+          git config --global core.autocrlf false
+          git config --global core.eol lf
+
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go }}
+          cache: true
+
+      - name: Build
+        run: |
+          go run build.go
+
+      - name: Test
+        # Our Windows tests currently don't work on Go 1.20
+        # https://github.com/syncthing/syncthing/issues/8779
+        # https://github.com/syncthing/syncthing/issues/8778
+        if: "!(matrix.go == '1.20' && matrix.runner == 'windows-latest')"
+        run: |
+          go run build.go test
+
+  #
+  # Meta checks for formatting, copyright, etc
+  #
+
+  correctness:
+    name: Check correctness
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Check correctness
+        run: |
+          go test -v ./meta
+
+  #
+  # Windows
+  #
+
+  package-windows:
+    name: Package for Windows
+    if: github.event_name == 'push' && github.ref == 'refs/heads/release'
+    environment: signing
+    needs:
+      - build-test
+    runs-on: windows-latest
+    steps:
+      - name: Set git to use LF
+        # Without this, the checkout will happen with CRLF line endings,
+        # which is fine for the source code but messes up tests that depend
+        # on data on disk being as expected. Ideally, those tests should be
+        # fixed, but not today.
+        run: |
+          git config --global core.autocrlf false
+          git config --global core.eol lf
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~\AppData\Local\go-build
+            ~\go\pkg\mod
+          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
+
+      - name: Install dependencies
+        run: |
+          go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@v1.4.0
+
+      - name: Create packages
+        run: |
+          go run build.go -goarch amd64 zip
+          go run build.go -goarch arm zip
+          go run build.go -goarch arm64 zip
+          go run build.go -goarch 386 zip
+        env:
+          CGO_ENABLED: "0"
+          CODESIGN_SIGNTOOL: ${{ secrets.CODESIGN_SIGNTOOL }}
+          CODESIGN_CERTIFICATE_BASE64: ${{ secrets.CODESIGN_CERTIFICATE_BASE64 }}
+          CODESIGN_CERTIFICATE_PASSWORD: ${{ secrets.CODESIGN_CERTIFICATE_PASSWORD }}
+          CODESIGN_TIMESTAMP_SERVER: ${{ secrets.CODESIGN_TIMESTAMP_SERVER }}
+
+      - name: Archive artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: packages
+          path: syncthing-windows-*.zip
+
+  #
+  # Linux
+  #
+
+  package-linux:
+    name: Package for Linux
+    needs:
+      - build-test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
+
+      - name: Create packages
+        run: |
+          archs=$(go tool dist list | grep linux | sed 's#linux/##')
+          for goarch in $archs ; do
+            go run build.go -goarch "$goarch" tar
+          done
+        env:
+          CGO_ENABLED: "0"
+
+      - name: Archive artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: packages
+          path: syncthing-linux-*.tar.gz
+
+  #
+  # macOS
+  #
+
+  package-macos:
+    name: Package for macOS
+    if: github.event_name == 'push' && github.ref == 'refs/heads/release'
+    environment: signing
+    needs:
+      - build-test
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
+
+      - name: Import signing certificate
+        run: |
+          # Set up a run-specific keychain, making it available for the
+          # `codesign` tool.
+          umask 066
+          KEYCHAIN_PATH=$RUNNER_TEMP/codesign.keychain
+          KEYCHAIN_PASSWORD=$(uuidgen)
+          security create-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
+          security default-keychain -s "$KEYCHAIN_PATH"
+          security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
+          security set-keychain-settings -lut 21600 "$KEYCHAIN_PATH"
+
+          # Import the certificate
+          CERTIFICATE_PATH=$RUNNER_TEMP/codesign.p12
+          echo "$DEVELOPER_ID_CERTIFICATE_BASE64" | base64 -d -o "$CERTIFICATE_PATH"
+          security import "$CERTIFICATE_PATH" -k "$KEYCHAIN_PATH" -P "$DEVELOPER_ID_CERTIFICATE_PASSWORD" -T /usr/bin/codesign -T /usr/bin/productsign
+          security set-key-partition-list -S apple-tool:,apple: -s -k actions "$KEYCHAIN_PATH"
+
+          # Set the codesign identity for following steps
+          echo "CODESIGN_IDENTITY=$CODESIGN_IDENTITY" >> $GITHUB_ENV
+        env:
+          DEVELOPER_ID_CERTIFICATE_BASE64: ${{ secrets.DEVELOPER_ID_CERTIFICATE_BASE64 }}
+          DEVELOPER_ID_CERTIFICATE_PASSWORD: ${{ secrets.DEVELOPER_ID_CERTIFICATE_PASSWORD }}
+          CODESIGN_IDENTITY: ${{ secrets.CODESIGN_IDENTITY }}
+
+      - name: Create package (amd64)
+        run: |
+          go run build.go -goarch amd64 zip
+        env:
+          CGO_ENABLED: "1"
+
+      - name: Create package (arm64 cross)
+        run: |
+          cat <<EOT > xgo.sh
+          #!/bin/bash
+          CGO_ENABLED=1 \
+            CGO_CFLAGS="-target arm64-apple-macos10.15" \
+            CGO_LDFLAGS="-target arm64-apple-macos10.15" \
+            go "\$@"
+          EOT
+          chmod 755 xgo.sh
+          go run build.go -gocmd ./xgo.sh -goarch arm64 zip
+        env:
+          CGO_ENABLED: "1"
+
+      - name: Create package (universal)
+        run: |
+          rm -rf _tmp
+          mkdir _tmp
+          pushd _tmp
+
+          unzip ../syncthing-macos-amd64-*.zip
+          unzip ../syncthing-macos-arm64-*.zip
+          lipo -create syncthing-macos-amd64-*/syncthing syncthing-macos-arm64-*/syncthing -o syncthing
+
+          amd64=(syncthing-macos-amd64-*)
+          universal="${amd64/amd64/universal}"
+          mv "$amd64" "$universal"
+          mv syncthing "$universal"
+          zip -r "../$universal.zip" "$universal"
+
+      - name: Archive artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: packages
+          path: syncthing-*.zip
+
+  #
+  # Cross compile other unixes
+  #
+
+  package-cross:
+    name: Package cross compiled
+    needs:
+      - build-test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-cross-${{ hashFiles('**/go.sum') }}
+
+      - name: Create packages
+        run: |
+          platforms=$(go tool dist list \
+            | grep -v aix/ppc64 \
+            | grep -v android/ \
+            | grep -v darwin/ \
+            | grep -v ios/ \
+            | grep -v js/ \
+            | grep -v linux/ \
+            | grep -v nacl/ \
+            | grep -v openbsd/arm\$ \
+            | grep -v openbsd/arm64 \
+            | grep -v openbsd/mips \
+            | grep -v plan9/ \
+            | grep -v windows/ \
+          )
+
+          for plat in $platforms; do
+            goos="${plat%/*}"
+            goarch="${plat#*/}"
+            go run build.go -goos "$goos" -goarch "$goarch" tar
+          done
+        env:
+          CGO_ENABLED: "0"
+
+      - name: Archive artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: packages
+          path: syncthing-*.tar.gz
+
+  #
+  # Source
+  #
+
+  package-source:
+    name: Package source code
+    needs:
+      - build-test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Package source
+        run: |
+          version=$(go run build.go version)
+          echo "$version" > RELEASE
+
+          go mod vendor
+          go run build.go assets
+
+          cd ..
+
+          tar c -z -f "syncthing-source-$version.tar.gz" \
+            --exclude .git \
+            syncthing
+
+          mv "syncthing-source-$version.tar.gz" syncthing
+
+      - name: Archive artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: packages
+          path: syncthing-source-*.tar.gz

.github/workflows/update-docs-translations.yaml

@@ -0,0 +1,28 @@
+name: Update translations and documentation
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '42 3 * * 1'
+
+jobs:
+
+  update_transifex_docs:
+    runs-on: ubuntu-latest
+    name: Update translations and documentation
+    steps:
+      - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
+      - uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # v2
+        with:
+          go-version: ^1.18.4
+      - run: |
+          set -euo pipefail
+          git config --global user.name 'Syncthing Release Automation'
+          git config --global user.email 'release@syncthing.net'
+          bash build.sh translate
+          bash build.sh prerelease
+          git push
+        env:
+          WEBLATE_TOKEN: ${{ secrets.WEBLATE_TOKEN }}

.gitignore

@@ -17,3 +17,6 @@ RELEASE
 deb
 *.bz2
 /repos
+/proto/scripts/protoc-gen-gosyncthing
+/gui/next-gen-gui
+.idea

.yamlfmt

@@ -0,0 +1,4 @@
+line_ending: lf
+formatter:
+  type: basic
+  retain_line_breaks: true

AUTHORS

@@ -18,14 +18,19 @@ Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.nor
 Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
 Alan Pope <alan@popey.com>
 Alberto Donato <albertodonato@users.noreply.github.com>
+Aleksey Vasenev <margtu-fivt@ya.ru>
 Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
+Alex Lindeman <139387+aelindeman@users.noreply.github.com>
 Alex Xu <alex.hello71@gmail.com>
 Alexander Graf (alex2108) <register-github@alex-graf.de>
+Alexandre Alves <alexandrealvesdb.contact@gmail.com>
 Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
 Aman Gupta <aman@tmm1.net>
 Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
+Andreas Sommer <andreas.sommer87@googlemail.com>
 andresvia <andres.via@gmail.com>
 Andrew Dunham (andrew-d) <andrew@du.nham.ca>
+Andrew Meyer <andrewm.bpi@gmail.com>
 Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
 Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
 André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
@@ -33,8 +38,10 @@ andyleap <andyleap@gmail.com>
 Anjan Momi <anjan@momi.ca>
 Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
 Antony Male (canton7) <antony.male@gmail.com>
+Anur <anurnomeru@163.com>
 Aranjedeath <Aranjedeath@users.noreply.github.com>
 Arkadiusz Tymiński <gevleeog@gmail.com>
+Aroun <login@b-vo.fr>
 Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
 Artur Zubilewicz <AkaZecik@users.noreply.github.com>
 Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
@@ -47,6 +54,7 @@ Ben Shepherd (benshep) <bjashepherd@gmail.com>
 Ben Sidhom (bsidhom) <bsidhom@gmail.com>
 Benedikt Heine (bebehei) <bebe@bebehei.de>
 Benedikt Morbach <benedikt.morbach@googlemail.com>
+Benjamin Nater <17193640+bn4t@users.noreply.github.com>
 Benno Fünfstück <benno.fuenfstueck@gmail.com>
 Benny Ng (tpng) <benny.tpng@gmail.com>
 boomsquared <54829195+boomsquared@users.noreply.github.com>
@@ -55,80 +63,113 @@ Boris Rybalkin <ribalkin@gmail.com>
 Brandon Philips (philips) <brandon@ifup.org>
 Brendan Long (brendanlong) <self@brendanlong.com>
 Brian R. Becker (brbecker) <brbecker@gmail.com>
+bt90 <btom1990@googlemail.com>
 Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
 Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
 Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
 Cedric Staniewski (xduugu) <cedric@gmx.ca>
 chenrui <rui@meetup.com>
+Chih-Hsuan Yen <yan12125@gmail.com>
+Choongkyu <choongkyu.kim+gh@gmail.com> <vapidlyrapid+gh@gmail.com>
 Chris Howie (cdhowie) <me@chrishowie.com>
 Chris Joel (cdata) <chris@scriptolo.gy>
 Chris Tonkinson <chris@masterbran.ch>
+Christian Prescott <me@christianprescott.com>
 chucic <chucic@seznam.cz>
 Colin Kennedy (moshen) <moshen.colin@gmail.com>
 Cromefire_ <tim.l@nghorst.net> <26320625+cromefire@users.noreply.github.com>
+cui fliter <imcusg@gmail.com>
 Cyprien Devillez <cypx@users.noreply.github.com>
 Dale Visser <dale.visser@live.com>
 Dan <benda.daniel@gmail.com>
+Daniel Barczyk <46358936+DanielBarczyk@users.noreply.github.com>
 Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
 Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
 Daniel Martí (mvdan) <mvdan@mvdan.cc>
 Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
 David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
+deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com>
 Denis A. (dva) <denisva@gmail.com>
 Dennis Wilson (snnd) <dw@risu.io>
 dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com> <27856297+dependabot-preview[bot]@users.noreply.github.com>
-dependabot[bot] <dependabot[bot]@users.noreply.github.com>
+dependabot[bot] <dependabot[bot]@users.noreply.github.com> <49699333+dependabot[bot]@users.noreply.github.com>
 derekriemer <derek.riemer@colorado.edu>
 desbma <desbma@users.noreply.github.com>
+Devon G. Redekopp <devon@redekopp.com>
+Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com>
 Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
 Domenic Horner <domenic@tgxn.net>
 Dominik Heidler (asdil12) <dominik@heidler.eu>
 Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
 Elliot Huffman <thelich2@gmail.com>
 Emil Hessman (ceh) <emil@hessman.se>
+Eng Zer Jun <engzerjun@gmail.com>
+entity0xfe <109791748+entity0xfe@users.noreply.github.com> <entity0xfe@my.domain>
+Eric Lesiuta <elesiuta@gmail.com>
+Eric P <eric@kastelo.net>
 Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
+Evan Spensley <94762716+0evan@users.noreply.github.com>
 Evgeny Kuznetsov <evgeny@kuznetsov.md>
 Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
 Felix Ableitner (Nutomic) <me@nutomic.com>
+Felix Lampe <mail@flampe.de>
 Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
 Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
 Frank Isemann (fti7) <frank@isemann.name>
+Gahl Saraf <saraf.gahl@gmail.com> <gahl@raftt.io>
 georgespatton <georgespatton@users.noreply.github.com>
 ghjklw <malo@jaffre.info>
 Gilli Sigurdsson (gillisig) <gilli@vx.is>
+Gleb Sinyavskiy <zhulik.gleb@gmail.com>
 Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
 greatroar <61184462+greatroar@users.noreply.github.com>
+Greg <gco@jazzhaiku.com>
 Han Boetes <han@boetes.org>
+HansK-p <42314815+HansK-p@users.noreply.github.com>
 Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
 Heiko Zuerker (Smiley73) <heiko@zuerker.org>
 Hugo Locurcio <hugo.locurcio@hugo.pro>
 Iain Barnett <iainspeed@gmail.com>
 Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
+ignacy123 <ignacy.buczek@onet.pl>
+Ikko Ashimine <eltociear@gmail.com>
 Ilya Brin <464157+ilyabrin@users.noreply.github.com>
 Iskander Sharipov (Alex) <quasilyte@gmail.com>
 Jaakko Hannikainen (jgke) <jgke@jgke.fi>
 Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
+Jack Croft <jccroft1@users.noreply.github.com>
 Jacob <jyundt@gmail.com>
 Jake Peterson (acogdev) <jake@acogdev.com>
 Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
+James O'Beirne <wild-github@au92.org>
 James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
 janost <janost@tuta.io>
+Jaroslav Lichtblau <svetlemodry@users.noreply.github.com>
 Jaroslav Malec (dzarda) <dzardacz@gmail.com>
 jaseg <githubaccount@jaseg.net>
+Jauder Ho <jauderho@users.noreply.github.com>
 Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
+Jaya Kumar <jaya.kumar@ict.nl>
+Jeffery To <jeffery.to@gmail.com>
 jelle van der Waa <jelle@vdwaa.nl>
 Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
 Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
+Jesse Lucas <jesse@jesselucas.com>
 Jochen Voss (seehuhn) <voss@seehuhn.de>
 Johan Andersson <j@i19.se>
 Johan Vromans (sciurius) <jvromans@squirrel.nl>
 John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
 Jonas Thelemann <e-mail@jonas-thelemann.de>
+Jonathan <artback@protonmail.com> <jonagn@gmail.com>
 Jonathan Cross <jcross@gmail.com>
+Jonta <359397+Jonta@users.noreply.github.com>
 Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
+jtagcat <git-514635f7@jtag.cat> <git-12dbd862@jtag.cat>
 Jörg Thalheim <Mic92@users.noreply.github.com>
+Jędrzej Kula <kula.jedrek@gmail.com>
 Kalle Laine <pahakalle@protonmail.com>
 Karol Różycki (krozycki) <rozycki.karol@gmail.com>
+Kebin Liu <lkebin@gmail.com>
 Keith Turner <kturner@apache.org>
 Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
 Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
@@ -138,13 +179,16 @@ Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
 klemens <ka7@github.com>
 Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
 Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
+Lars Lehtonen <lars.lehtonen@gmail.com>
 Laurent Arnoud <laurent@spkdev.net>
 Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
 Leo Arias (elopio) <yo@elopio.net>
 Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
 Lode Hoste (Zillode) <zillode@zillode.be>
 Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
+LSmithx2 <42276854+lsmithx2@users.noreply.github.com>
 Lukas Lihotzki <lukas@lihotzki.de>
+luzpaz <luzpaz@users.noreply.github.com>
 Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
 Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
 Marc Pujol (kilburn) <kilburn@la3.org>
@@ -153,6 +197,7 @@ marco-m <marco.molteni@laposte.net>
 Marcus Legendre <marcus.legendre@gmail.com>
 Mario Majila <mariustshipichik@gmail.com>
 Mark Pulford (mpx) <mark@kyne.com.au>
+Martchus <martchus@gmx.net>
 Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
 Mateusz Ż <thedead4fun@live.com>
 Matic Potočnik <hairyfotr@gmail.com>
@@ -160,30 +205,37 @@ Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
 Matt Robenolt <matt@ydekproductions.com>
 Matteo Ruina <matteo.ruina@gmail.com>
 Maurizio Tomasi <ziotom78@gmail.com>
+Max <github@germancoding.com>
 Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
 MaximAL <almaximal@ya.ru>
 Maxime Thirouin <m@moox.io>
+mclang <1721600+mclang@users.noreply.github.com>
 Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
 Michael Ploujnikov (plouj) <ploujj@gmail.com>
 Michael Rienstra <mrienstra@gmail.com>
 Michael Tilli (pyfisch) <pyfisch@gmail.com>
+MichaIng <micha@dietpi.com>
 Mike Boone <mike@boonedocks.net>
 MikeLund <MikeLund@users.noreply.github.com>
 MikolajTwarog <43782609+MikolajTwarog@users.noreply.github.com>
 Mingxuan Lin <gdlmx@users.noreply.github.com>
 mv1005 <49659413+mv1005@users.noreply.github.com>
 Nate Morrison (nrm21) <natemorrison@gmail.com>
+Naveen <172697+naveensrinivasan@users.noreply.github.com>
 Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
+Nick Busey <NickBusey@users.noreply.github.com>
 Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
 Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
 Nicolas Perraut <n.perraut@gmail.com>
 Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
 Nils Jakobi (thunderstorm99) <jakobi.nils@gmail.com>
+NinoM4ster <ninom4ster@gmail.com>
 Nitroretro <43112364+Nitroretro@users.noreply.github.com>
 NoLooseEnds <jon.koslung@gmail.com>
 Oliver Freyermuth <o.freyermuth@googlemail.com>
 otbutz <tbutz@optitool.de>
 Otiel <Otiel@users.noreply.github.com>
+overkill <22098433+0verk1ll@users.noreply.github.com>
 Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
 Pablo <pbaeyens31+github@gmail.com>
 Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
@@ -195,37 +247,49 @@ Peter Badida <KeyWeeUsr@users.noreply.github.com>
 Peter Dave Hello <hsu@peterdavehello.org>
 Peter Hoeg (peterhoeg) <peter@speartail.com>
 Peter Marquardt (wwwutz) <wwwutz@gmail.com> <wwwutz@googlemail.com>
+Phani Rithvij <phanirithvij2000@gmail.com>
 Phil Davis <phil.davis@inf.org>
 Philippe Schommers (filoozoom) <philippe@schommers.be>
 Phill Luby (pluby) <phill.luby@newredo.com>
 Pier Paolo Ramon <ramonpierre@gmail.com>
 Piotr Bejda (piobpl) <piotrb10@gmail.com>
 Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
+Quentin Hibon <qh.public@yahoo.com>
+Rahmi Pruitt <rjpruitt16@gmail.com>
+red_led <red-led@users.noreply.github.com>
 Richard Hartmann <RichiH@users.noreply.github.com>
 Robert Carosi (nov1n) <robert@carosi.nl>
+Roberto Santalla <roobre@users.noreply.github.com>
 Robin Schoonover <robin@cornhooves.org>
 Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
 Ross Smith II (rasa) <ross@smithii.com>
 rubenbe <github-com-00ff86@vandamme.email>
 Ruslan Yevdokymov <38809160+ruslanye@users.noreply.github.com>
+Ryan Qian <i@bitbili.net>
 Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
 Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
 Scott Klupfel (kluppy) <kluppy@going2blue.com>
+sec65 <106604020+sec65@users.noreply.github.com>
 Sergey Mishin (ralder) <ralder@yandex.ru>
+Shaarad Dalvi <60266155+shaaraddalvi@users.noreply.github.com> <shdalv@microsoft.com>
 Simon Frei (imsodin) <freisim93@gmail.com>
 Simon Mwepu <simonmwepu@gmail.com>
 Sly_tom_cat <slytomcat@mail.ru>
 Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
 Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
+Steven Eckhoff <steven.eckhoff.opensource@gmail.com>
 Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
+Syncthing Automation <automation@syncthing.net>
+Syncthing Release Automation <release@syncthing.net>
 Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
 Thomas Hipp <thomashipp@gmail.com>
 Tim Abell (timabell) <tim@timwise.co.uk>
 Tim Howes (timhowes) <timhowes@berkeley.edu>
+Tobias Klauser <tobias.klauser@gmail.com>
 Tobias Nygren (tnn2) <tnn@nygren.pp.se>
 Tobias Tom (tobiastom) <t.tom@succont.de>
 Tom Jakubowski <tom@crystae.net>
-Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com>
+Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com> <twilczynski@naver.com>
 Tommy Thorn <tommy-github-email@thorn.ws>
 Tully Robinson (tojrobinson) <tully@tojr.org>
 Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
@@ -233,12 +297,17 @@ Tyler Kropp <kropptyler@gmail.com>
 Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
 Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
 Victor Buinsky (buinsky) <vix_booja@tut.by>
+Vik <63919734+ViktorOn@users.noreply.github.com>
 Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
-Vladimir Rusinov <vrusinov@google.com>
+villekalliomaki <53118179+villekalliomaki@users.noreply.github.com>
+Vladimir Rusinov <vrusinov@google.com> <vladimir.rusinov@gmail.com>
 wangguoliang <liangcszzu@163.com>
 William A. Kennington III (wkennington) <william@wkennington.com>
+wouter bolsterlee <wouter@bolsterl.ee>
 Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
+xarx00 <xarx00@users.noreply.github.com>
 Xavier O. (damajor) <damajor@gmail.com>
 xjtdy888 (xjtdy888) <xjtdy888@163.com>
 Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
 佛跳墙 <daoquan@qq.com>
+落心 <luoxin.ttt@gmail.com>

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 ## Reporting Bugs
 
-Please file bugs in the [Github Issue
+Please file bugs in the [GitHub Issue
 Tracker](https://github.com/syncthing/syncthing/issues). Include at
 least the following:
 
@@ -24,10 +24,15 @@ too much information will never get you yelled at. :)
 ## Contributing Translations
 
 All translations are done via
-[Transifex](https://www.transifex.com/projects/p/syncthing/). If you
+[Weblate](https://hosted.weblate.org/projects/syncthing/). If you wish
-wish to contribute to a translation, just head over there and sign up.
+to contribute to a translation, just head over there and sign up.
 Before every release, the language resources are updated from the
-latest info on Transifex.
+latest info on Weblate.
+
+Note that the previously used service at
+[Transifex](https://www.transifex.com/projects/p/syncthing/) is being
+retired and we kindly ask you to sign up on Weblate for continued
+involvement.
 
 ## Contributing Code
 

Dockerfile

@@ -11,16 +11,20 @@ RUN rm -f syncthing && go run build.go -no-upgrade build syncthing
 
 FROM alpine
 
-EXPOSE 8384 22000 21027/udp
+EXPOSE 8384 22000/tcp 22000/udp 21027/udp
 
 VOLUME ["/var/syncthing"]
 
-RUN apk add --no-cache ca-certificates su-exec
+RUN apk add --no-cache ca-certificates curl libcap su-exec tzdata
 
 COPY --from=builder /src/syncthing /bin/syncthing
 COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
 
 ENV PUID=1000 PGID=1000 HOME=/var/syncthing
 
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -o --color=never OK || exit 1
+
 ENV STGUIADDRESS=0.0.0.0:8384
+RUN chmod 755 /bin/entrypoint.sh
 ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]

Dockerfile.builder

@@ -6,4 +6,4 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 	locales rubygems ruby-dev build-essential git \
 	&& apt-get clean \
 	&& rm -rf /var/lib/apt/lists/* \
-	&& gem install --no-ri --no-rdoc fpm
+	&& gem install fpm

Dockerfile.buildx

@@ -0,0 +1,19 @@
+FROM alpine
+ARG TARGETARCH
+
+EXPOSE 8384 22000/tcp 22000/udp 21027/udp
+
+VOLUME ["/var/syncthing"]
+
+RUN apk add --no-cache ca-certificates curl libcap su-exec tzdata
+
+COPY ./syncthing-linux-$TARGETARCH /bin/syncthing
+COPY ./script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/syncthing
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -o --color=never OK || exit 1
+
+ENV STGUIADDRESS=0.0.0.0:8384
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]

Dockerfile.stcrashreceiver

@@ -0,0 +1,18 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f stcrashreceiver && go run build.go build stcrashreceiver
+
+FROM alpine
+
+EXPOSE 8080
+
+COPY --from=builder /src/stcrashreceiver /bin/stcrashreceiver
+
+ENTRYPOINT [ "/bin/stcrashreceiver" ]

Dockerfile.strelaypoolsrv

@@ -0,0 +1,26 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f strelaysrv && go run build.go -no-upgrade build strelaypoolsrv
+
+FROM alpine
+
+EXPOSE 8080
+
+RUN apk add --no-cache ca-certificates su-exec curl
+ENV PUID=1000 PGID=1000 MAXMIND_KEY=
+
+RUN mkdir /var/strelaypoolsrv && chown 1000 /var/strelaypoolsrv
+USER 1000
+
+COPY --from=builder /src/strelaypoolsrv /bin/strelaypoolsrv
+COPY --from=builder /src/script/strelaypoolsrv-entrypoint.sh /bin/entrypoint.sh
+
+WORKDIR /var/strelaypoolsrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaypoolsrv", "-listen", ":8080"]

Dockerfile.stupgrades

@@ -0,0 +1,23 @@
+ARG GOVERSION=latest
+FROM golang:$GOVERSION AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f stupgrades && go run build.go build stupgrades
+
+FROM alpine
+
+EXPOSE 8080
+
+COPY --from=builder /src/stupgrades /bin/stupgrades
+
+ENTRYPOINT [ \
+	"/bin/stupgrades", \
+	"-f", "/nightly.json->https://build.syncthing.net/guestAuth/repository/download/Release_Nightly/.lastSuccessful/nightly.json", \
+	"-f", "/syncthing-macos/appcast.xml->https://build.syncthing.net/guestAuth/repository/download/SyncthingMacOS_CreateAppcastXml/.lastSuccessful/appcast.xml" \
+	]
+

GOALS.md

@@ -57,7 +57,7 @@ latest technology is not always available to any given individual.
 > Computers include desktops, laptops, servers, virtual machines, small
 > general purpose computers such as Raspberry Pis and, *where possible*,
 > tablets and phones. NAS appliances, toasters, cars, firearms, thermostats
-> and so on may include computing capabitilies but it is not our goal for
+> and so on may include computing capabilities but it is not our goal for
 > Syncthing to run smoothly on these devices.
 
 ### 6. For Individuals

README-Docker.md

@@ -7,25 +7,57 @@ Use the `/var/syncthing` volume to have the synchronized files available on the
 host. You can add more folders and map them as you prefer.
 
 Note that Syncthing runs as UID 1000 and GID 1000 by default. These may be
-altered with the ``PUID`` and ``PGID`` environment variables.
+altered with the `PUID` and `PGID` environment variables. In addition
+the name of the Syncthing instance can be optionally defined by using
+`--hostname=syncthing` parameter.
+
+To grant Syncthing additional capabilities without running as root, use the
+`PCAP` environment variable with the same syntax as that for `setcap(8)`.
+For example, `PCAP=cap_chown,cap_fowner+ep`.
 
 ## Example Usage
 
+**Docker cli**
 ```
 $ docker pull syncthing/syncthing
-$ docker run -p 8384:8384 -p 22000:22000 \
+$ docker run -p 8384:8384 -p 22000:22000/tcp -p 22000:22000/udp -p 21027:21027/udp \
     -v /wherever/st-sync:/var/syncthing \
+    --hostname=my-syncthing \
     syncthing/syncthing:latest
 ```
 
+**Docker compose**
+```yml
+---
+version: "3"
+services:
+  syncthing:
+    image: syncthing/syncthing
+    container_name: syncthing
+    hostname: my-syncthing
+    environment:
+      - PUID=1000
+      - PGID=1000
+    volumes:
+      - /wherever/st-sync:/var/syncthing
+    ports:
+      - 8384:8384 # Web UI
+      - 22000:22000/tcp # TCP file transfers
+      - 22000:22000/udp # QUIC file transfers
+      - 21027:21027/udp # Receive local discovery broadcasts
+    restart: unless-stopped
+```
+
 ## Discovery
 
-Note that local device discovery will not work with the above command,
+Note that Docker's default network mode prevents local IP addresses from
-resulting in poor local transfer rates if local device addresses are not
+being discovered, as Syncthing is only able to see the internal IP of the
-manually configured.
+container on the `172.17.0.0/16` subnet. This will result in poor transfer rates
+if local device addresses are not manually configured.
 
-To allow local discovery, the docker host network can be used instead:
+It is therefore advisable to use the [host network mode](https://docs.docker.com/network/host/) instead:
 
+**Docker cli**
 ```
 $ docker pull syncthing/syncthing
 $ docker run --network=host \
@@ -33,6 +65,24 @@ $ docker run --network=host \
     syncthing/syncthing:latest
 ```
 
+**Docker compose**
+```yml
+---
+version: "3"
+services:
+  syncthing:
+    image: syncthing/syncthing
+    container_name: syncthing
+    hostname: my-syncthing
+    environment:
+      - PUID=1000
+      - PGID=1000
+    volumes:
+      - /wherever/st-sync:/var/syncthing
+    network_mode: host
+    restart: unless-stopped
+```
+
 Be aware that syncthing alone is now in control of what interfaces and ports it
 listens on. You can edit the syncthing configuration to change the defaults if
 there are conflicts.

README.md

@@ -73,15 +73,16 @@ This helps the team understand what are the biggest pain points for our users, a
 
 ## Getting in Touch
 
-The first and best point of contact is the [Forum][8]. There is also an IRC
+The first and best point of contact is the [Forum][8].
-channel, `#syncthing` on [freenode][4] (with a [web client][9]), for talking
+If you've found something that is clearly a
-directly to developers and users. If you've found something that is clearly a
 bug, feel free to report it in the [GitHub issue tracker][10].
 
 ## Building
 
-Building Syncthing from source is easy, and there's a [guide][5]
+Building Syncthing from source is easy. After extracting the source bundle from
-that describes it for both Unix and Windows systems.
+a release or checking out git, you just need to run `go run build.go` and the
+binaries are created in `./bin`. There's [a guide][5] with more details on the
+build process.
 
 ## Signed Releases
 
@@ -95,24 +96,22 @@ binaries are also properly code signed.
 
 ## Documentation
 
-Please see the [Syncthing documentation site][6].
+Please see the Syncthing [documentation site][6] [[source]][17].
 
 All code is licensed under the [MPLv2 License][7].
 
 [1]: https://docs.syncthing.net/specs/bep-v1.html
 [2]: https://docs.syncthing.net/intro/getting-started.html
-[3]: https://github.com/syncthing/syncthing/blob/master/etc
+[3]: https://github.com/syncthing/syncthing/blob/main/etc
-[4]: https://www.freenode.net/
 [5]: https://docs.syncthing.net/dev/building.html
 [6]: https://docs.syncthing.net/
-[7]: https://github.com/syncthing/syncthing/blob/master/LICENSE
+[7]: https://github.com/syncthing/syncthing/blob/main/LICENSE
 [8]: https://forum.syncthing.net/
-[9]: https://kiwiirc.com/client/irc.freenode.net/#syncthing
 [10]: https://github.com/syncthing/syncthing/issues
 [11]: https://docs.syncthing.net/users/contrib.html#gui-wrappers
 [12]: https://www.bountysource.com/teams/syncthing/issues
-[13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
+[13]: https://github.com/syncthing/syncthing/blob/main/GOALS.md
 [14]: assets/logo-text-128.png
 [15]: https://syncthing.net/
-[16]: https://github.com/syncthing/syncthing/blob/master/README-Docker.md
+[16]: https://github.com/syncthing/syncthing/blob/main/README-Docker.md
-
+[17]: https://github.com/syncthing/docs

build.go

@@ -4,6 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
+//go:build ignore
 // +build ignore
 
 package main
@@ -14,13 +15,12 @@ import (
 	"bytes"
 	"compress/flate"
 	"compress/gzip"
-	"crypto/sha256"
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -30,25 +30,34 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"text/template"
 	"time"
+
+	buildpkg "github.com/syncthing/syncthing/lib/build"
 )
 
 var (
-	versionRe     = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
+	goarch         string
-	goarch        string
+	goos           string
-	goos          string
+	noupgrade      bool
-	noupgrade     bool
+	version        string
-	version       string
+	goCmd          string
-	goCmd         string
+	race           bool
-	race          bool
+	debug          = os.Getenv("BUILDDEBUG") != ""
-	debug         = os.Getenv("BUILDDEBUG") != ""
+	extraTags      string
-	extraTags     string
+	installSuffix  string
-	installSuffix string
+	pkgdir         string
-	pkgdir        string
+	cc             string
-	cc            string
+	run            string
-	debugBinary   bool
+	benchRun       string
-	coverage      bool
+	buildOut       string
-	timeout       = "120s"
+	debugBinary    bool
+	coverage       bool
+	long           bool
+	timeout        = "120s"
+	longTimeout    = "600s"
+	numVersions    = 5
+	withNextGenGUI = os.Getenv("BUILD_NEXT_GEN_GUI") != ""
 )
 
 type target struct {
@@ -56,12 +65,11 @@ type target struct {
 	debname           string
 	debdeps           []string
 	debpre            string
-	debpost           string
 	description       string
 	buildPkgs         []string
 	binaryName        string
 	archiveFiles      []archiveFile
-	systemdServices   []string
+	systemdService    string
 	installationFiles []archiveFile
 	tags              []string
 }
@@ -83,7 +91,6 @@ var targets = map[string]target{
 		name:        "syncthing",
 		debname:     "syncthing",
 		debdeps:     []string{"libc6", "procps"},
-		debpost:     "script/post-upgrade",
 		description: "Open Source Continuous File Synchronization",
 		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/syncthing"},
 		binaryName:  "syncthing", // .exe will be added automatically for Windows builds
@@ -94,6 +101,7 @@ var targets = map[string]target{
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 			// All files from etc/ and extra/ added automatically in init().
 		},
+		systemdService: "syncthing@*.service",
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
 			{src: "README.md", dst: "deb/usr/share/doc/syncthing/README.txt", perm: 0644},
@@ -112,6 +120,7 @@ var targets = map[string]target{
 			{src: "etc/linux-systemd/system/syncthing@.service", dst: "deb/lib/systemd/system/syncthing@.service", perm: 0644},
 			{src: "etc/linux-systemd/system/syncthing-resume.service", dst: "deb/lib/systemd/system/syncthing-resume.service", perm: 0644},
 			{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0644},
+			{src: "etc/linux-sysctl/30-syncthing.conf", dst: "deb/usr/lib/sysctl.d/30-syncthing.conf", perm: 0644},
 			{src: "etc/firewall-ufw/syncthing", dst: "deb/etc/ufw/applications.d/syncthing", perm: 0644},
 			{src: "etc/linux-desktop/syncthing-start.desktop", dst: "deb/usr/share/applications/syncthing-start.desktop", perm: 0644},
 			{src: "etc/linux-desktop/syncthing-ui.desktop", dst: "deb/usr/share/applications/syncthing-ui.desktop", perm: 0644},
@@ -137,15 +146,14 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
-		systemdServices: []string{
+		systemdService: "stdiscosrv.service",
-			"cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service",
-		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
 			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0644},
 			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0644},
 			{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service", dst: "deb/lib/systemd/system/stdiscosrv.service", perm: 0644},
 			{src: "cmd/stdiscosrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-discosrv", perm: 0644},
 			{src: "cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv", dst: "deb/etc/ufw/applications.d/stdiscosrv", perm: 0644},
 		},
@@ -166,9 +174,7 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
-		systemdServices: []string{
+		systemdService: "strelaysrv.service",
-			"cmd/strelaysrv/etc/linux-systemd/strelaysrv.service",
-		},
 		installationFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
 			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0644},
@@ -176,6 +182,7 @@ var targets = map[string]target{
 			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0644},
 			{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0644},
+			{src: "cmd/strelaysrv/etc/linux-systemd/strelaysrv.service", dst: "deb/lib/systemd/system/strelaysrv.service", perm: 0644},
 			{src: "cmd/strelaysrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-relaysrv", perm: 0644},
 			{src: "cmd/strelaysrv/etc/firewall-ufw/strelaysrv", dst: "deb/etc/ufw/applications.d/strelaysrv", perm: 0644},
 		},
@@ -200,21 +207,21 @@ var targets = map[string]target{
 			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/AUTHORS.txt", perm: 0644},
 		},
 	},
+	"stupgrades": {
+		name:        "stupgrades",
+		description: "Syncthing Upgrade Check Server",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/stupgrades"},
+		binaryName:  "stupgrades",
+	},
+	"stcrashreceiver": {
+		name:        "stupgrastcrashreceiverdes",
+		description: "Syncthing Crash Server",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/stcrashreceiver"},
+		binaryName:  "stcrashreceiver",
+	},
 }
 
-// These are repos we need to clone to run "go generate"
+func initTargets() {
-
-type dependencyRepo struct {
-	path   string
-	repo   string
-	commit string
-}
-
-var dependencyRepos = []dependencyRepo{
-	{path: "xdr", repo: "https://github.com/calmh/xdr.git", commit: "08e072f9cb16"},
-}
-
-func init() {
 	all := targets["all"]
 	pkgs, _ := filepath.Glob("cmd/*")
 	for _, pkg := range pkgs {
@@ -223,6 +230,9 @@ func init() {
 			// ignore dotfiles
 			continue
 		}
+		if noupgrade && pkg == "stupgrades" {
+			continue
+		}
 		all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/cmd/%s", pkg))
 	}
 	targets["all"] = all
@@ -254,6 +264,8 @@ func main() {
 		}()
 	}
 
+	initTargets()
+
 	// Invoking build.go with no parameters at all builds everything (incrementally),
 	// which is what you want for maximum error checking during development.
 	if flag.NArg() == 0 {
@@ -276,47 +288,58 @@ func main() {
 }
 
 func runCommand(cmd string, target target) {
+	var tags []string
+	if noupgrade {
+		tags = []string{"noupgrade"}
+	}
+	tags = append(tags, strings.Fields(extraTags)...)
+
 	switch cmd {
 	case "install":
-		var tags []string
-		if noupgrade {
-			tags = []string{"noupgrade"}
-		}
-		tags = append(tags, strings.Fields(extraTags)...)
 		install(target, tags)
 		metalintShort()
 
 	case "build":
-		var tags []string
-		if noupgrade {
-			tags = []string{"noupgrade"}
-		}
-		tags = append(tags, strings.Fields(extraTags)...)
 		build(target, tags)
 
 	case "test":
-		test("github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
+		test(strings.Fields(extraTags), "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
 
 	case "bench":
-		bench("github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
+		bench(strings.Fields(extraTags), "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
+
+	case "integration":
+		integration(false)
+
+	case "integrationbench":
+		integration(true)
 
 	case "assets":
 		rebuildAssets()
 
+	case "update-deps":
+		updateDependencies()
+
 	case "proto":
 		proto()
 
+	case "testmocks":
+		testmocks()
+
 	case "translate":
 		translate()
 
 	case "transifex":
 		transifex()
 
+	case "weblate":
+		weblate()
+
 	case "tar":
-		buildTar(target)
+		buildTar(target, tags)
 
 	case "zip":
-		buildZip(target)
+		buildZip(target, tags)
 
 	case "deb":
 		buildDeb(target)
@@ -333,6 +356,20 @@ func runCommand(cmd string, target target) {
 	case "version":
 		fmt.Println(getVersion())
 
+	case "changelog":
+		vers, err := currentAndLatestVersions(numVersions)
+		if err != nil {
+			log.Fatal(err)
+		}
+		for _, ver := range vers {
+			underline := strings.Repeat("=", len(ver))
+			msg, err := tagMessage(ver)
+			if err != nil {
+				log.Fatal(err)
+			}
+			fmt.Printf("%s\n%s\n\n%s\n\n", ver, underline, msg)
+		}
+
 	default:
 		log.Fatalf("Unknown command %q", cmd)
 	}
@@ -351,17 +388,30 @@ func parseFlags() {
 	flag.StringVar(&cc, "cc", os.Getenv("CC"), "Set CC environment variable for `go build`")
 	flag.BoolVar(&debugBinary, "debug-binary", debugBinary, "Create unoptimized binary to use with delve, set -gcflags='-N -l' and omit -ldflags")
 	flag.BoolVar(&coverage, "coverage", coverage, "Write coverage profile of tests to coverage.txt")
+	flag.BoolVar(&long, "long", long, "Run tests without the -short flag")
+	flag.IntVar(&numVersions, "num-versions", numVersions, "Number of versions for changelog command")
+	flag.StringVar(&run, "run", "", "Specify which tests to run")
+	flag.StringVar(&benchRun, "bench", "", "Specify which benchmarks to run")
+	flag.BoolVar(&withNextGenGUI, "with-next-gen-gui", withNextGenGUI, "Also build 'newgui'")
+	flag.StringVar(&buildOut, "build-out", "", "Set the '-o' value for 'go build'")
 	flag.Parse()
 }
 
-func test(pkgs ...string) {
+func test(tags []string, pkgs ...string) {
 	lazyRebuildAssets()
 
-	args := []string{"test", "-short", "-timeout", timeout, "-tags", "purego"}
+	tags = append(tags, "purego")
+	args := []string{"test", "-tags", strings.Join(tags, " ")}
+	if long {
+		timeout = longTimeout
+	} else {
+		args = append(args, "-short")
+	}
+	args = append(args, "-timeout", timeout)
 
 	if runtime.GOARCH == "amd64" {
 		switch runtime.GOOS {
-		case "darwin", "linux", "freebsd": // , "windows": # See https://github.com/golang/go/issues/27089
+		case buildpkg.Darwin, buildpkg.Linux, buildpkg.FreeBSD: // , "windows": # See https://github.com/golang/go/issues/27089
 			args = append(args, "-race")
 		}
 	}
@@ -370,15 +420,56 @@ func test(pkgs ...string) {
 		args = append(args, "-covermode", "atomic", "-coverprofile", "coverage.txt", "-coverpkg", strings.Join(pkgs, ","))
 	}
 
+	args = append(args, runArgs()...)
+
 	runPrint(goCmd, append(args, pkgs...)...)
 }
 
-func bench(pkgs ...string) {
+func bench(tags []string, pkgs ...string) {
 	lazyRebuildAssets()
-	runPrint(goCmd, append([]string{"test", "-run", "NONE", "-bench", "."}, pkgs...)...)
+	args := append([]string{"test", "-run", "NONE", "-tags", strings.Join(tags, " ")}, benchArgs()...)
+	runPrint(goCmd, append(args, pkgs...)...)
+}
+
+func integration(bench bool) {
+	lazyRebuildAssets()
+	args := []string{"test", "-v", "-timeout", "60m", "-tags"}
+	tags := "purego,integration"
+	if bench {
+		tags += ",benchmark"
+	}
+	args = append(args, tags)
+	args = append(args, runArgs()...)
+	if bench {
+		if run == "" {
+			args = append(args, "-run", "Benchmark")
+		}
+		args = append(args, benchArgs()...)
+	}
+	args = append(args, "./test")
+	fmt.Println(args)
+	runPrint(goCmd, args...)
+}
+
+func runArgs() []string {
+	if run == "" {
+		return nil
+	}
+	return []string{"-run", run}
+}
+
+func benchArgs() []string {
+	if benchRun == "" {
+		return []string{"-bench", "."}
+	}
+	return []string{"-bench", benchRun}
 }
 
 func install(target target, tags []string) {
+	if (target.name == "syncthing" || target.name == "") && !withNextGenGUI {
+		log.Println("Notice: Next generation GUI will not be built; see --with-next-gen-gui.")
+	}
+
 	lazyRebuildAssets()
 
 	tags = append(target.tags, tags...)
@@ -389,9 +480,7 @@ func install(target target, tags []string) {
 	}
 	os.Setenv("GOBIN", filepath.Join(cwd, "bin"))
 
-	os.Setenv("GOOS", goos)
+	setBuildEnvVars()
-	os.Setenv("GOARCH", goarch)
-	os.Setenv("CC", cc)
 
 	// On Windows generate a special file which the Go compiler will
 	// automatically use when generating Windows binaries to set things like
@@ -410,14 +499,16 @@ func install(target target, tags []string) {
 }
 
 func build(target target, tags []string) {
+	if (target.name == "syncthing" || target.name == "") && !withNextGenGUI {
+		log.Println("Notice: Next generation GUI will not be built; see --with-next-gen-gui.")
+	}
+
 	lazyRebuildAssets()
 	tags = append(target.tags, tags...)
 
 	rmr(target.BinaryName())
 
-	os.Setenv("GOOS", goos)
+	setBuildEnvVars()
-	os.Setenv("GOARCH", goarch)
-	os.Setenv("CC", cc)
 
 	// On Windows generate a special file which the Go compiler will
 	// automatically use when generating Windows binaries to set things like
@@ -435,10 +526,26 @@ func build(target target, tags []string) {
 	}
 
 	args := []string{"build", "-v"}
+	if buildOut != "" {
+		args = append(args, "-o", buildOut)
+	}
 	args = appendParameters(args, tags, target.buildPkgs...)
 	runPrint(goCmd, args...)
 }
 
+func setBuildEnvVars() {
+	os.Setenv("GOOS", goos)
+	os.Setenv("GOARCH", goarch)
+	os.Setenv("CC", cc)
+	if os.Getenv("CGO_ENABLED") == "" {
+		switch goos {
+		case "darwin", "solaris":
+		default:
+			os.Setenv("CGO_ENABLED", "0")
+		}
+	}
+}
+
 func appendParameters(args []string, tags []string, pkgs ...string) []string {
 	if pkgdir != "" {
 		args = append(args, "-pkgdir", pkgdir)
@@ -455,26 +562,27 @@ func appendParameters(args []string, tags []string, pkgs ...string) []string {
 
 	if !debugBinary {
 		// Regular binaries get version tagged and skip some debug symbols
-		args = append(args, "-ldflags", ldflags())
+		args = append(args, "-trimpath", "-ldflags", ldflags(tags))
 	} else {
 		// -gcflags to disable optimizations and inlining. Skip -ldflags
 		// because `Could not launch program: decoding dwarf section info at
 		// offset 0x0: too short` on 'dlv exec ...' see
-		// https://github.com/derekparker/delve/issues/79
+		// https://github.com/go-delve/delve/issues/79
-		args = append(args, "-gcflags", "-N -l")
+		args = append(args, "-gcflags", "all=-N -l")
 	}
 
 	return append(args, pkgs...)
 }
 
-func buildTar(target target) {
+func buildTar(target target, tags []string) {
 	name := archiveName(target)
 	filename := name + ".tar.gz"
 
-	var tags []string
+	for _, tag := range tags {
-	if noupgrade {
+		if tag == "noupgrade" {
-		tags = []string{"noupgrade"}
+			name += "-noupgrade"
-		name += "-noupgrade"
+			break
+		}
 	}
 
 	build(target, tags)
@@ -490,14 +598,15 @@ func buildTar(target target) {
 	fmt.Println(filename)
 }
 
-func buildZip(target target) {
+func buildZip(target target, tags []string) {
 	name := archiveName(target)
 	filename := name + ".zip"
 
-	var tags []string
+	for _, tag := range tags {
-	if noupgrade {
+		if tag == "noupgrade" {
-		tags = []string{"noupgrade"}
+			name += "-noupgrade"
-		name += "-noupgrade"
+			break
+		}
 	}
 
 	build(target, tags)
@@ -565,11 +674,13 @@ func buildDeb(target target) {
 	for _, dep := range target.debdeps {
 		args = append(args, "-d", dep)
 	}
-	for _, service := range target.systemdServices {
+	if target.systemdService != "" {
-		args = append(args, "--deb-systemd", service)
+		debpost, err := createPostInstScript(target)
-	}
+		defer os.Remove(debpost)
-	if target.debpost != "" {
+		if err != nil {
-		args = append(args, "--after-upgrade", target.debpost)
+			log.Fatal(err)
+		}
+		args = append(args, "--after-upgrade", debpost)
 	}
 	if target.debpre != "" {
 		args = append(args, "--before-install", target.debpre)
@@ -577,6 +688,28 @@ func buildDeb(target target) {
 	runPrint("fpm", args...)
 }
 
+func createPostInstScript(target target) (string, error) {
+	scriptname := filepath.Join("script", "deb-post-inst.template")
+	t, err := template.ParseFiles(scriptname)
+	if err != nil {
+		return "", err
+	}
+	scriptname = strings.TrimSuffix(scriptname, ".template")
+	w, err := os.Create(scriptname)
+	if err != nil {
+		return "", err
+	}
+	defer w.Close()
+	if err = t.Execute(w, struct {
+		Service, Command string
+	}{
+		target.systemdService, target.binaryName,
+	}); err != nil {
+		return "", err
+	}
+	return scriptname, nil
+}
+
 func shouldBuildSyso(dir string) (string, error) {
 	type M map[string]interface{}
 	version := getVersion()
@@ -596,11 +729,14 @@ func shouldBuildSyso(dir string) (string, error) {
 			},
 		},
 		"StringFileInfo": M{
-			"FileDescription": "Open Source Continuous File Synchronization",
+			"CompanyName":      "The Syncthing Authors",
-			"LegalCopyright":  "The Syncthing Authors",
+			"FileDescription":  "Syncthing - Open Source Continuous File Synchronization",
-			"FileVersion":     version,
+			"FileVersion":      version,
-			"ProductVersion":  version,
+			"InternalName":     "syncthing",
-			"ProductName":     "Syncthing",
+			"LegalCopyright":   "The Syncthing Authors",
+			"OriginalFilename": "syncthing",
+			"ProductName":      "Syncthing",
+			"ProductVersion":   version,
 		},
 		"IconPath": "assets/logo.ico",
 	})
@@ -609,7 +745,7 @@ func shouldBuildSyso(dir string) (string, error) {
 	}
 
 	jsonPath := filepath.Join(dir, "versioninfo.json")
-	err = ioutil.WriteFile(jsonPath, bs, 0644)
+	err = os.WriteFile(jsonPath, bs, 0644)
 	if err != nil {
 		return "", errors.New("failed to create " + jsonPath + ": " + err.Error())
 	}
@@ -622,7 +758,13 @@ func shouldBuildSyso(dir string) (string, error) {
 
 	sysoPath := filepath.Join(dir, "cmd", "syncthing", "resource.syso")
 
-	if _, err := runError("goversioninfo", "-o", sysoPath); err != nil {
+	// See https://github.com/josephspurrier/goversioninfo#command-line-flags
+	armOption := ""
+	if strings.Contains(goarch, "arm") {
+		armOption = "-arm=true"
+	}
+
+	if _, err := runError("goversioninfo", "-o", sysoPath, armOption); err != nil {
 		return "", errors.New("failed to create " + sysoPath + ": " + err.Error())
 	}
 
@@ -642,12 +784,12 @@ func shouldCleanupSyso(sysoFilePath string) {
 // exists. The permission bits are copied as well. If dst already exists and
 // the contents are identical to src the modification time is not updated.
 func copyFile(src, dst string, perm os.FileMode) error {
-	in, err := ioutil.ReadFile(src)
+	in, err := os.ReadFile(src)
 	if err != nil {
 		return err
 	}
 
-	out, err := ioutil.ReadFile(dst)
+	out, err := os.ReadFile(dst)
 	if err != nil {
 		// The destination probably doesn't exist, we should create
 		// it.
@@ -663,7 +805,7 @@ func copyFile(src, dst string, perm os.FileMode) error {
 
 copy:
 	os.MkdirAll(filepath.Dir(dst), 0777)
-	if err := ioutil.WriteFile(dst, in, perm); err != nil {
+	if err := os.WriteFile(dst, in, perm); err != nil {
 		return err
 	}
 
@@ -691,11 +833,46 @@ func rebuildAssets() {
 }
 
 func lazyRebuildAssets() {
-	if shouldRebuildAssets("lib/api/auto/gui.files.go", "gui") || shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.files.go", "cmd/strelaypoolsrv/auto/gui") {
+	shouldRebuild := shouldRebuildAssets("lib/api/auto/gui.files.go", "gui") ||
+		shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.files.go", "cmd/strelaypoolsrv/gui")
+
+	if withNextGenGUI {
+		shouldRebuild = buildNextGenGUI() || shouldRebuild
+	}
+
+	if shouldRebuild {
 		rebuildAssets()
 	}
 }
 
+func buildNextGenGUI() bool {
+	// Check if we need to run the npm process, and if so also set the flag
+	// to rebuild Go assets afterwards. The index.html is regenerated every
+	// time by the build process. This assumes the new GUI ends up in
+	// next-gen-gui/dist/next-gen-gui.
+
+	if !shouldRebuildAssets("gui/next-gen-gui/index.html", "next-gen-gui") {
+		// The GUI is up to date.
+		return false
+	}
+
+	runPrintInDir("next-gen-gui", "npm", "install")
+	runPrintInDir("next-gen-gui", "npm", "run", "build", "--", "--prod", "--subresource-integrity")
+
+	rmr("gui/tech-ui")
+
+	for _, src := range listFiles("next-gen-gui/dist") {
+		rel, _ := filepath.Rel("next-gen-gui/dist", src)
+		dst := filepath.Join("gui", rel)
+		if err := copyFile(src, dst, 0644); err != nil {
+			fmt.Println("copy:", err)
+			os.Exit(1)
+		}
+	}
+
+	return true
+}
+
 func shouldRebuildAssets(target, srcdir string) bool {
 	info, err := os.Stat(target)
 	if err != nil {
@@ -722,24 +899,57 @@ func shouldRebuildAssets(target, srcdir string) bool {
 	return assetsAreNewer
 }
 
+func updateDependencies() {
+	// Figure out desired Go version
+	bs, err := os.ReadFile("go.mod")
+	if err != nil {
+		log.Fatal(err)
+	}
+	re := regexp.MustCompile(`(?m)^go\s+([0-9.]+)`)
+	matches := re.FindSubmatch(bs)
+	if len(matches) != 2 {
+		log.Fatal("failed to parse go.mod")
+	}
+	goVersion := string(matches[1])
+
+	runPrint(goCmd, "get", "-u", "./...")
+	runPrint(goCmd, "mod", "tidy", "-go="+goVersion, "-compat="+goVersion)
+
+	// We might have updated the protobuf package and should regenerate to match.
+	proto()
+}
+
 func proto() {
 	pv := protobufVersion()
-	dependencyRepos = append(dependencyRepos,
+	repo := "https://github.com/gogo/protobuf.git"
-		dependencyRepo{path: "protobuf", repo: "https://github.com/gogo/protobuf.git", commit: pv},
+	path := filepath.Join("repos", "protobuf")
-	)
 
-	runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
+	runPrint(goCmd, "install", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
 	os.MkdirAll("repos", 0755)
-	for _, dep := range dependencyRepos {
+
-		path := filepath.Join("repos", dep.path)
+	if _, err := os.Stat(path); err != nil {
-		if _, err := os.Stat(path); err != nil {
+		runPrint("git", "clone", repo, path)
-			runPrintInDir("repos", "git", "clone", dep.repo, dep.path)
+	} else {
-		} else {
+		runPrintInDir(path, "git", "fetch")
-			runPrintInDir(path, "git", "fetch")
-		}
-		runPrintInDir(path, "git", "checkout", dep.commit)
 	}
-	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/stdiscosrv")
+	runPrintInDir(path, "git", "checkout", pv)
+
+	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/cmd/stdiscosrv")
+	runPrint(goCmd, "generate", "proto/generate.go")
+}
+
+func testmocks() {
+	args := []string{
+		"generate",
+		"github.com/syncthing/syncthing/lib/config",
+		"github.com/syncthing/syncthing/lib/connections",
+		"github.com/syncthing/syncthing/lib/discover",
+		"github.com/syncthing/syncthing/lib/events",
+		"github.com/syncthing/syncthing/lib/logger",
+		"github.com/syncthing/syncthing/lib/model",
+		"github.com/syncthing/syncthing/lib/protocol",
+	}
+	runPrint(goCmd, args...)
 }
 
 func translate() {
@@ -758,13 +968,19 @@ func transifex() {
 	runPrint(goCmd, "run", "../../../../script/transifexdl.go")
 }
 
-func ldflags() string {
+func weblate() {
+	os.Chdir("gui/default/assets/lang")
+	runPrint(goCmd, "run", "../../../../script/weblatedl.go")
+}
+
+func ldflags(tags []string) string {
 	b := new(strings.Builder)
 	b.WriteString("-w")
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Version=%s", version)
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp=%d", buildStamp())
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User=%s", buildUser())
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Host=%s", buildHost())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Tags=%s", strings.Join(tags, ","))
 	if v := os.Getenv("EXTRA_LDFLAGS"); v != "" {
 		fmt.Fprintf(b, " %s", v)
 	}
@@ -781,7 +997,7 @@ func rmr(paths ...string) {
 }
 
 func getReleaseVersion() (string, error) {
-	bs, err := ioutil.ReadFile("RELEASE")
+	bs, err := os.ReadFile("RELEASE")
 	if err != nil {
 		return "", err
 	}
@@ -789,15 +1005,39 @@ func getReleaseVersion() (string, error) {
 }
 
 func getGitVersion() (string, error) {
-	v, err := runError("git", "describe", "--always", "--dirty")
+	// The current version as Git sees it
+	bs, err := runError("git", "describe", "--always", "--dirty", "--abbrev=8")
 	if err != nil {
 		return "", err
 	}
-	v = versionRe.ReplaceAllFunc(v, func(s []byte) []byte {
+	vcur := string(bs)
-		s[0] = '+'
+
-		return s
+	// The closest current tag name
-	})
+	bs, err = runError("git", "describe", "--always", "--abbrev=0")
-	return string(v), nil
+	if err != nil {
+		return "", err
+	}
+	v0 := string(bs)
+
+	// To be more semantic-versionish and ensure proper ordering in our
+	// upgrade process, we make sure there's only one hyphen in the version.
+
+	versionRe := regexp.MustCompile(`-([0-9]{1,3}-g[0-9a-f]{5,10}(-dirty)?)`)
+	if m := versionRe.FindStringSubmatch(vcur); len(m) > 0 {
+		suffix := strings.ReplaceAll(m[1], "-", ".")
+
+		if strings.Contains(v0, "-") {
+			// We're based of a tag with a prerelease string. We can just
+			// add our dev stuff directly.
+			return fmt.Sprintf("%s.dev.%s", v0, suffix), nil
+		}
+
+		// We're based on a release version. We need to bump the patch
+		// version and then add a -dev prerelease string.
+		next := nextPatchVersion(v0)
+		return fmt.Sprintf("%s-dev.%s", next, suffix), nil
+	}
+	return vcur, nil
 }
 
 func getVersion() string {
@@ -868,7 +1108,7 @@ func getBranchSuffix() string {
 
 	branch = parts[len(parts)-1]
 	switch branch {
-	case "master", "release":
+	case "master", "release", "main":
 		// these are not special
 		return ""
 	}
@@ -1089,11 +1329,11 @@ func zipFile(out string, files []archiveFile) {
 
 		if strings.HasSuffix(f.dst, ".txt") {
 			// Text file. Read it and convert line endings.
-			bs, err := ioutil.ReadAll(sf)
+			bs, err := io.ReadAll(sf)
 			if err != nil {
 				log.Fatal(err)
 			}
-			bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\n', '\r'}, -1)
+			bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\r', '\n'}, -1)
 			fh.UncompressedSize = uint32(len(bs))
 			fh.UncompressedSize64 = uint64(len(bs))
 
@@ -1164,6 +1404,33 @@ func windowsCodesign(file string) {
 		args := []string{"sign", "/fd", algo}
 		if f := os.Getenv("CODESIGN_CERTIFICATE_FILE"); f != "" {
 			args = append(args, "/f", f)
+		} else if b := os.Getenv("CODESIGN_CERTIFICATE_BASE64"); b != "" {
+			// Decode the PFX certificate from base64.
+			bs, err := base64.RawStdEncoding.DecodeString(b)
+			if err != nil {
+				log.Println("Codesign: signing failed: decoding base64:", err)
+				return
+			}
+
+			// Write it to a temporary file
+			f, err := os.CreateTemp("", "codesign-*.pfx")
+			if err != nil {
+				log.Println("Codesign: signing failed: creating temp file:", err)
+				return
+			}
+			_ = f.Chmod(0600) // best effort remove other users' access
+			defer os.Remove(f.Name())
+			if _, err := f.Write(bs); err != nil {
+				log.Println("Codesign: signing failed: writing temp file:", err)
+				return
+			}
+			if err := f.Close(); err != nil {
+				log.Println("Codesign: signing failed: closing temp file:", err)
+				return
+			}
+
+			// Use that when signing
+			args = append(args, "/f", f.Name())
 		}
 		if p := os.Getenv("CODESIGN_CERTIFICATE_PASSWORD"); p != "" {
 			args = append(args, "/p", p)
@@ -1183,7 +1450,7 @@ func windowsCodesign(file string) {
 
 		bs, err := runError(st, args...)
 		if err != nil {
-			log.Println("Codesign: signing failed:", string(bs))
+			log.Printf("Codesign: signing failed: %v: %s", err, string(bs))
 			return
 		}
 		log.Println("Codesign: successfully signed", file, "using", algo)
@@ -1200,32 +1467,6 @@ func metalintShort() {
 	runPrint(goCmd, "test", "-short", "-run", "Metalint", "./meta")
 }
 
-func temporaryBuildDir() (string, error) {
-	// The base of our temp dir is "syncthing-xxxxxxxx" where the x:es
-	// are eight bytes from the sha256 of our working directory. We do
-	// this because we want a name in the global temp dir that doesn't
-	// conflict with someone else building syncthing on the same
-	// machine, yet is persistent between runs from the same source
-	// directory.
-	wd, err := os.Getwd()
-	if err != nil {
-		return "", err
-	}
-	hash := sha256.Sum256([]byte(wd))
-	base := fmt.Sprintf("syncthing-%x", hash[:4])
-
-	// The temp dir is taken from $STTMPDIR if set, otherwise the system
-	// default (potentially infrluenced by $TMPDIR on unixes).
-	var tmpDir string
-	if t := os.Getenv("STTMPDIR"); t != "" {
-		tmpDir = t
-	} else {
-		tmpDir = os.TempDir()
-	}
-
-	return filepath.Join(tmpDir, base), nil
-}
-
 func (t target) BinaryName() string {
 	if goos == "windows" {
 		return t.binaryName + ".exe"
@@ -1240,3 +1481,70 @@ func protobufVersion() string {
 	}
 	return string(bs)
 }
+
+func currentAndLatestVersions(n int) ([]string, error) {
+	bs, err := runError("git", "tag", "--sort", "taggerdate")
+	if err != nil {
+		return nil, err
+	}
+
+	lines := strings.Split(string(bs), "\n")
+	reverseStrings(lines)
+
+	// The one at the head is the latest version. We always keep that one.
+	// Then we filter out remaining ones with dashes (pre-releases etc).
+
+	latest := lines[:1]
+	nonPres := filterStrings(lines[1:], func(s string) bool { return !strings.Contains(s, "-") })
+	vers := append(latest, nonPres...)
+	return vers[:n], nil
+}
+
+func reverseStrings(ss []string) {
+	for i := 0; i < len(ss)/2; i++ {
+		ss[i], ss[len(ss)-1-i] = ss[len(ss)-1-i], ss[i]
+	}
+}
+
+func filterStrings(ss []string, op func(string) bool) []string {
+	n := ss[:0]
+	for _, s := range ss {
+		if op(s) {
+			n = append(n, s)
+		}
+	}
+	return n
+}
+
+func tagMessage(tag string) (string, error) {
+	hash, err := runError("git", "rev-parse", tag)
+	if err != nil {
+		return "", err
+	}
+	obj, err := runError("git", "cat-file", "-p", string(hash))
+	if err != nil {
+		return "", err
+	}
+	return trimTagMessage(string(obj), tag), nil
+}
+
+func trimTagMessage(msg, tag string) string {
+	firstBlank := strings.Index(msg, "\n\n")
+	if firstBlank > 0 {
+		msg = msg[firstBlank+2:]
+	}
+	msg = strings.TrimPrefix(msg, tag)
+	beginSig := strings.Index(msg, "-----BEGIN PGP")
+	if beginSig > 0 {
+		msg = msg[:beginSig]
+	}
+	return strings.TrimSpace(msg)
+}
+
+func nextPatchVersion(ver string) string {
+	parts := strings.SplitN(ver, "-", 2)
+	digits := strings.Split(parts[0], ".")
+	n, _ := strconv.Atoi(digits[len(digits)-1])
+	digits[len(digits)-1] = strconv.Itoa(n + 1)
+	return strings.Join(digits, ".")
+}

build.ps1

@@ -0,0 +1,20 @@
+function build {
+    go run build.go @args
+}
+
+$cmd, $rest = $args
+switch ($cmd) {
+    "test" {
+        $env:LOGGER_DISCARD=1
+        build test
+    }
+
+    "bench" {
+        $env:LOGGER_DISCARD=1
+        build bench
+    }
+
+    default {
+        build @rest
+    }
+}

build.sh

@@ -2,8 +2,6 @@
 set -euo pipefail
 IFS=$'\n\t'
 
-STTRACE=${STTRACE:-}
-
 script() {
 	name="$1"
 	shift
@@ -15,88 +13,23 @@ build() {
 }
 
 case "${1:-default}" in
-	default)
-		build
-		;;
-
-	clean)
-		build "$@"
-		;;
-
-	tar)
-		build "$@"
-		;;
-
-	assets)
-		build "$@"
-		;;
-
-	xdr)
-		build "$@"
-		;;
-
-	translate)
-		build "$@"
-		;;
-
-	deb)
-		build "$@"
-		;;
-
-	setup)
-		build "$@"
-		;;
-
 	test)
 		LOGGER_DISCARD=1 build test
 		;;
 
 	bench)
-		LOGGER_DISCARD=1 build bench | script benchfilter
+		LOGGER_DISCARD=1 build bench
 		;;
 
 	prerelease)
-		go run script/authors.go
+		script authors
-		build transifex
+		build weblate
 		pushd man ; ./refresh.sh ; popd
 		git add -A gui man AUTHORS
 		git commit -m 'gui, man, authors: Update docs, translations, and contributors'
 		;;
 
-	noupgrade)
-		build -no-upgrade tar
-		;;
-
-	all)
-		platforms=(
-			darwin-amd64 dragonfly-amd64 freebsd-amd64 linux-amd64 netbsd-amd64 openbsd-amd64 solaris-amd64 windows-amd64
-			freebsd-386 linux-386 netbsd-386 openbsd-386 windows-386
-			linux-arm linux-arm64 linux-ppc64 linux-ppc64le
-		)
-
-		for plat in "${platforms[@]}"; do
-			echo Building "$plat"
-
-			goos="${plat%-*}"
-			goarch="${plat#*-}"
-			dist="tar"
-
-			if [[ $goos == "windows" ]]; then
-				dist="zip"
-			fi
-
-			build -goos "$goos" -goarch "$goarch" "$dist"
-			echo
-		done
-		;;
-
-	test-xunit)
-
-		(GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -v -race ./lib/... ./cmd/... || true) > tests.out
-		go2xunit -output tests.xml -fail < tests.out
-		;;
-
 	*)
-		echo "Unknown build command $1"
+		build "$@"
 		;;
 esac

cmd/stcli/client.go

@@ -1,96 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"context"
-	"crypto/tls"
-	"errors"
-	"fmt"
-	"net"
-	"net/http"
-	"strings"
-
-	"github.com/syncthing/syncthing/lib/config"
-)
-
-type APIClient struct {
-	http.Client
-	cfg    config.GUIConfiguration
-	apikey string
-}
-
-func getClient(cfg config.GUIConfiguration) *APIClient {
-	httpClient := http.Client{
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: true,
-			},
-			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
-				return net.Dial(cfg.Network(), cfg.Address())
-			},
-		},
-	}
-	return &APIClient{
-		Client: httpClient,
-		cfg:    cfg,
-		apikey: cfg.APIKey,
-	}
-}
-
-func (c *APIClient) Endpoint() string {
-	if c.cfg.Network() == "unix" {
-		return "http://unix/"
-	}
-	url := c.cfg.URL()
-	if !strings.HasSuffix(url, "/") {
-		url += "/"
-	}
-	return url
-}
-
-func (c *APIClient) Do(req *http.Request) (*http.Response, error) {
-	req.Header.Set("X-API-Key", c.apikey)
-	resp, err := c.Client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	return resp, checkResponse(resp)
-}
-
-func (c *APIClient) Get(url string) (*http.Response, error) {
-	request, err := http.NewRequest("GET", c.Endpoint()+"rest/"+url, nil)
-	if err != nil {
-		return nil, err
-	}
-	return c.Do(request)
-}
-
-func (c *APIClient) Post(url, body string) (*http.Response, error) {
-	request, err := http.NewRequest("POST", c.Endpoint()+"rest/"+url, bytes.NewBufferString(body))
-	if err != nil {
-		return nil, err
-	}
-	return c.Do(request)
-}
-
-func checkResponse(response *http.Response) error {
-	if response.StatusCode == 404 {
-		return errors.New("invalid endpoint or API call")
-	} else if response.StatusCode == 403 {
-		return errors.New("invalid API key")
-	} else if response.StatusCode != 200 {
-		data, err := responseToBArray(response)
-		if err != nil {
-			return err
-		}
-		body := strings.TrimSpace(string(data))
-		return fmt.Errorf("unexpected HTTP status returned: %s\n%s", response.Status, body)
-	}
-	return nil
-}

cmd/stcli/main.go

@@ -1,192 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bufio"
-	"crypto/tls"
-	"encoding/json"
-	"flag"
-	"log"
-	"os"
-	"reflect"
-
-	"github.com/AudriusButkevicius/recli"
-	"github.com/flynn-archive/go-shlex"
-	"github.com/mattn/go-isatty"
-	"github.com/pkg/errors"
-	"github.com/syncthing/syncthing/lib/build"
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/events"
-	"github.com/syncthing/syncthing/lib/locations"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/urfave/cli"
-)
-
-func main() {
-	// This is somewhat a hack around a chicken and egg problem.
-	// We need to set the home directory and potentially other flags to know where the syncthing instance is running
-	// in order to get it's config ... which we then use to construct the actual CLI ... at which point it's too late
-	// to add flags there...
-	homeBaseDir := locations.GetBaseDir(locations.ConfigBaseDir)
-	guiCfg := config.GUIConfiguration{}
-
-	flags := flag.NewFlagSet("", flag.ContinueOnError)
-	flags.StringVar(&guiCfg.RawAddress, "gui-address", guiCfg.RawAddress, "Override GUI address (e.g. \"http://192.0.2.42:8443\")")
-	flags.StringVar(&guiCfg.APIKey, "gui-apikey", guiCfg.APIKey, "Override GUI API key")
-	flags.StringVar(&homeBaseDir, "home", homeBaseDir, "Set configuration directory")
-
-	// Implement the same flags at the lower CLI, with the same default values (pre-parse), but do nothing with them.
-	// This is so that we could reuse os.Args
-	fakeFlags := []cli.Flag{
-		cli.StringFlag{
-			Name:  "gui-address",
-			Value: guiCfg.RawAddress,
-			Usage: "Override GUI address (e.g. \"http://192.0.2.42:8443\")",
-		},
-		cli.StringFlag{
-			Name:  "gui-apikey",
-			Value: guiCfg.APIKey,
-			Usage: "Override GUI API key",
-		},
-		cli.StringFlag{
-			Name:  "home",
-			Value: homeBaseDir,
-			Usage: "Set configuration directory",
-		},
-	}
-
-	// Do not print usage of these flags, and ignore errors as this can't understand plenty of things
-	flags.Usage = func() {}
-	_ = flags.Parse(os.Args[1:])
-
-	// Now if the API key and address is not provided (we are not connecting to a remote instance),
-	// try to rip it out of the config.
-	if guiCfg.RawAddress == "" && guiCfg.APIKey == "" {
-		// Update the base directory
-		err := locations.SetBaseDir(locations.ConfigBaseDir, homeBaseDir)
-		if err != nil {
-			log.Fatal(errors.Wrap(err, "setting home"))
-		}
-
-		// Load the certs and get the ID
-		cert, err := tls.LoadX509KeyPair(
-			locations.Get(locations.CertFile),
-			locations.Get(locations.KeyFile),
-		)
-		if err != nil {
-			log.Fatal(errors.Wrap(err, "reading device ID"))
-		}
-
-		myID := protocol.NewDeviceID(cert.Certificate[0])
-
-		// Load the config
-		cfg, err := config.Load(locations.Get(locations.ConfigFile), myID, events.NoopLogger)
-		if err != nil {
-			log.Fatalln(errors.Wrap(err, "loading config"))
-		}
-
-		guiCfg = cfg.GUI()
-	} else if guiCfg.Address() == "" || guiCfg.APIKey == "" {
-		log.Fatalln("Both -gui-address and -gui-apikey should be specified")
-	}
-
-	if guiCfg.Address() == "" {
-		log.Fatalln("Could not find GUI Address")
-	}
-
-	if guiCfg.APIKey == "" {
-		log.Fatalln("Could not find GUI API key")
-	}
-
-	client := getClient(guiCfg)
-
-	cfg, err := getConfig(client)
-	original := cfg.Copy()
-	if err != nil {
-		log.Fatalln(errors.Wrap(err, "getting config"))
-	}
-
-	// Copy the config and set the default flags
-	recliCfg := recli.DefaultConfig
-	recliCfg.IDTag.Name = "xml"
-	recliCfg.SkipTag.Name = "json"
-
-	commands, err := recli.New(recliCfg).Construct(&cfg)
-	if err != nil {
-		log.Fatalln(errors.Wrap(err, "config reflect"))
-	}
-
-	// Construct the actual CLI
-	app := cli.NewApp()
-	app.Name = "stcli"
-	app.HelpName = app.Name
-	app.Author = "The Syncthing Authors"
-	app.Usage = "Syncthing command line interface"
-	app.Version = build.Version
-	app.Flags = fakeFlags
-	app.Metadata = map[string]interface{}{
-		"client": client,
-	}
-	app.Commands = []cli.Command{
-		{
-			Name:        "config",
-			HideHelp:    true,
-			Usage:       "Configuration modification command group",
-			Subcommands: commands,
-		},
-		showCommand,
-		operationCommand,
-		errorsCommand,
-	}
-
-	tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
-	if !tty {
-		// Not a TTY, consume from stdin
-		scanner := bufio.NewScanner(os.Stdin)
-		for scanner.Scan() {
-			input, err := shlex.Split(scanner.Text())
-			if err != nil {
-				log.Fatalln(errors.Wrap(err, "parsing input"))
-			}
-			if len(input) == 0 {
-				continue
-			}
-			err = app.Run(append(os.Args, input...))
-			if err != nil {
-				log.Fatalln(err)
-			}
-		}
-		err = scanner.Err()
-		if err != nil {
-			log.Fatalln(err)
-		}
-	} else {
-		err = app.Run(os.Args)
-		if err != nil {
-			log.Fatalln(err)
-		}
-	}
-
-	if !reflect.DeepEqual(cfg, original) {
-		body, err := json.MarshalIndent(cfg, "", "  ")
-		if err != nil {
-			log.Fatalln(err)
-		}
-		resp, err := client.Post("system/config", string(body))
-		if err != nil {
-			log.Fatalln(err)
-		}
-		if resp.StatusCode != 200 {
-			body, err := responseToBArray(resp)
-			if err != nil {
-				log.Fatalln(err)
-			}
-			log.Fatalln(string(body))
-		}
-	}
-}

cmd/stcli/utils.go

@@ -1,94 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"os"
-
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/urfave/cli"
-)
-
-func responseToBArray(response *http.Response) ([]byte, error) {
-	bytes, err := ioutil.ReadAll(response.Body)
-	if err != nil {
-		return nil, err
-	}
-	return bytes, response.Body.Close()
-}
-
-func emptyPost(url string) cli.ActionFunc {
-	return func(c *cli.Context) error {
-		client := c.App.Metadata["client"].(*APIClient)
-		_, err := client.Post(url, "")
-		return err
-	}
-}
-
-func dumpOutput(url string) cli.ActionFunc {
-	return func(c *cli.Context) error {
-		client := c.App.Metadata["client"].(*APIClient)
-		response, err := client.Get(url)
-		if err != nil {
-			return err
-		}
-		return prettyPrintResponse(c, response)
-	}
-}
-
-func getConfig(c *APIClient) (config.Configuration, error) {
-	cfg := config.Configuration{}
-	response, err := c.Get("system/config")
-	if err != nil {
-		return cfg, err
-	}
-	bytes, err := responseToBArray(response)
-	if err != nil {
-		return cfg, err
-	}
-	err = json.Unmarshal(bytes, &cfg)
-	if err == nil {
-		return cfg, err
-	}
-	return cfg, nil
-}
-
-func expects(n int, actionFunc cli.ActionFunc) cli.ActionFunc {
-	return func(ctx *cli.Context) error {
-		if ctx.NArg() != n {
-			plural := ""
-			if n != 1 {
-				plural = "s"
-			}
-			return fmt.Errorf("expected %d argument%s, got %d", n, plural, ctx.NArg())
-		}
-		return actionFunc(ctx)
-	}
-}
-
-func prettyPrintJSON(data interface{}) error {
-	enc := json.NewEncoder(os.Stdout)
-	enc.SetIndent("", "  ")
-	return enc.Encode(data)
-}
-
-func prettyPrintResponse(c *cli.Context, response *http.Response) error {
-	bytes, err := responseToBArray(response)
-	if err != nil {
-		return err
-	}
-	var data interface{}
-	if err := json.Unmarshal(bytes, &data); err != nil {
-		return err
-	}
-	// TODO: Check flag for pretty print format
-	return prettyPrintJSON(data)
-}

cmd/stcompdirs/main.go

@@ -7,7 +7,6 @@
 package main
 
 import (
-	"crypto/md5"
 	"errors"
 	"flag"
 	"fmt"
@@ -15,6 +14,8 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 func main() {
@@ -74,7 +75,7 @@ type fileInfo struct {
 	name string
 	mode os.FileMode
 	mod  int64
-	hash [16]byte
+	hash [sha256.Size]byte
 }
 
 func (f fileInfo) String() string {
@@ -106,11 +107,7 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 			if err != nil {
 				return err
 			}
-			h := md5.New()
+			f.hash = sha256.Sum256([]byte(tgt))
-			h.Write([]byte(tgt))
-			hash := h.Sum(nil)
-
-			copy(f.hash[:], hash)
 		} else if info.IsDir() {
 			f = fileInfo{
 				name: rn,
@@ -123,7 +120,7 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 				mode: info.Mode(),
 				mod:  info.ModTime().Unix(),
 			}
-			sum, err := md5file(path)
+			sum, err := sha256file(path)
 			if err != nil {
 				return err
 			}
@@ -150,14 +147,14 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 	return errc
 }
 
-func md5file(fname string) (hash [16]byte, err error) {
+func sha256file(fname string) (hash [sha256.Size]byte, err error) {
 	f, err := os.Open(fname)
 	if err != nil {
 		return
 	}
 	defer f.Close()
 
-	h := md5.New()
+	h := sha256.New()
 	io.Copy(h, f)
 	hb := h.Sum(nil)
 	copy(hash[:], hb)

cmd/stcrashreceiver/diskstore.go

@@ -0,0 +1,187 @@
+// Copyright (C) 2023 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+)
+
+type diskStore struct {
+	dir      string
+	inbox    chan diskEntry
+	maxBytes int64
+	maxFiles int
+
+	currentFiles []currentFile
+	currentSize  int64
+}
+
+type diskEntry struct {
+	path string
+	data []byte
+}
+
+type currentFile struct {
+	path  string
+	size  int64
+	mtime int64
+}
+
+func (d *diskStore) Serve(ctx context.Context) {
+	if err := os.MkdirAll(d.dir, 0750); err != nil {
+		log.Println("Creating directory:", err)
+		return
+	}
+
+	if err := d.inventory(); err != nil {
+		log.Println("Failed to inventory disk store:", err)
+	}
+	d.clean()
+
+	cleanTimer := time.NewTicker(time.Minute)
+	inventoryTimer := time.NewTicker(24 * time.Hour)
+
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	for {
+		select {
+		case entry := <-d.inbox:
+			path := d.fullPath(entry.path)
+
+			if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+				log.Println("Creating directory:", err)
+				continue
+			}
+
+			buf.Reset()
+			gw.Reset(buf)
+			if _, err := gw.Write(entry.data); err != nil {
+				log.Println("Failed to compress crash report:", err)
+				continue
+			}
+			if err := gw.Close(); err != nil {
+				log.Println("Failed to compress crash report:", err)
+				continue
+			}
+			if err := os.WriteFile(path, buf.Bytes(), 0644); err != nil {
+				log.Printf("Failed to write %s: %v", entry.path, err)
+				_ = os.Remove(path)
+				continue
+			}
+
+			d.currentSize += int64(buf.Len())
+			d.currentFiles = append(d.currentFiles, currentFile{
+				size: int64(len(entry.data)),
+				path: path,
+			})
+
+		case <-cleanTimer.C:
+			d.clean()
+
+		case <-inventoryTimer.C:
+			if err := d.inventory(); err != nil {
+				log.Println("Failed to inventory disk store:", err)
+			}
+
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (d *diskStore) Put(path string, data []byte) bool {
+	select {
+	case d.inbox <- diskEntry{
+		path: path,
+		data: data,
+	}:
+		return true
+	default:
+		return false
+	}
+}
+
+func (d *diskStore) Get(path string) ([]byte, error) {
+	path = d.fullPath(path)
+	bs, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	gr, err := gzip.NewReader(bytes.NewReader(bs))
+	if err != nil {
+		return nil, err
+	}
+	defer gr.Close()
+	return io.ReadAll(gr)
+}
+
+func (d *diskStore) Exists(path string) bool {
+	path = d.fullPath(path)
+	_, err := os.Lstat(path)
+	return err == nil
+}
+
+func (d *diskStore) clean() {
+	for len(d.currentFiles) > 0 && (len(d.currentFiles) > d.maxFiles || d.currentSize > d.maxBytes) {
+		f := d.currentFiles[0]
+		log.Println("Removing", f.path)
+		if err := os.Remove(f.path); err != nil {
+			log.Println("Failed to remove file:", err)
+		}
+		d.currentFiles = d.currentFiles[1:]
+		d.currentSize -= f.size
+	}
+	var oldest time.Duration
+	if len(d.currentFiles) > 0 {
+		oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
+	}
+	log.Printf("Clean complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
+}
+
+func (d *diskStore) inventory() error {
+	d.currentFiles = nil
+	d.currentSize = 0
+	err := filepath.Walk(d.dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		if filepath.Ext(path) != ".gz" {
+			return nil
+		}
+		d.currentSize += info.Size()
+		d.currentFiles = append(d.currentFiles, currentFile{
+			path:  path,
+			size:  info.Size(),
+			mtime: info.ModTime().Unix(),
+		})
+		return nil
+	})
+	sort.Slice(d.currentFiles, func(i, j int) bool {
+		return d.currentFiles[i].mtime < d.currentFiles[j].mtime
+	})
+	var oldest time.Duration
+	if len(d.currentFiles) > 0 {
+		oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
+	}
+	log.Printf("Inventory complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
+	return err
+}
+
+func (d *diskStore) fullPath(path string) string {
+	return filepath.Join(d.dir, path[0:2], path[2:]) + ".gz"
+}

cmd/stcrashreceiver/main.go

@@ -0,0 +1,151 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Command stcrashreceiver is a trivial HTTP server that allows two things:
+//
+// - uploading files (crash reports) named like a SHA256 hash using a PUT request
+// - checking whether such file exists using a HEAD request
+//
+// Typically this should be deployed behind something that manages HTTPS.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/alecthomas/kong"
+	"github.com/syncthing/syncthing/lib/sha256"
+	"github.com/syncthing/syncthing/lib/ur"
+
+	raven "github.com/getsentry/raven-go"
+)
+
+const maxRequestSize = 1 << 20 // 1 MiB
+
+type cli struct {
+	Dir           string        `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."`
+	DSN           string        `help:"Sentry DSN" env:"SENTRY_DSN"`
+	Listen        string        `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"`
+	MaxDiskFiles  int           `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"`
+	MaxDiskSizeMB int64         `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"`
+	CleanInterval time.Duration `help:"Interval between cleaning up old reports" default:"12h" env:"CLEAN_INTERVAL"`
+	SentryQueue   int           `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"`
+	DiskQueue     int           `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"`
+}
+
+func main() {
+	var params cli
+	kong.Parse(&params)
+
+	mux := http.NewServeMux()
+
+	ds := &diskStore{
+		dir:      filepath.Join(params.Dir, "crash_reports"),
+		inbox:    make(chan diskEntry, params.DiskQueue),
+		maxFiles: params.MaxDiskFiles,
+		maxBytes: params.MaxDiskSizeMB << 20,
+	}
+	go ds.Serve(context.Background())
+
+	ss := &sentryService{
+		dsn:   params.DSN,
+		inbox: make(chan sentryRequest, params.SentryQueue),
+	}
+	go ss.Serve(context.Background())
+
+	cr := &crashReceiver{
+		store:  ds,
+		sentry: ss,
+	}
+
+	mux.Handle("/", cr)
+
+	if params.DSN != "" {
+		mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports")))
+	}
+
+	log.SetOutput(os.Stdout)
+	if err := http.ListenAndServe(params.Listen, mux); err != nil {
+		log.Fatalln("HTTP serve:", err)
+	}
+}
+
+func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *http.Request) {
+	return func(w http.ResponseWriter, req *http.Request) {
+		lr := io.LimitReader(req.Body, maxRequestSize)
+		bs, err := io.ReadAll(lr)
+		req.Body.Close()
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+
+		var reports []ur.FailureReport
+		err = json.Unmarshal(bs, &reports)
+		if err != nil {
+			http.Error(w, err.Error(), 400)
+			return
+		}
+		if len(reports) == 0 {
+			// Shouldn't happen
+			log.Printf("Got zero failure reports")
+			return
+		}
+
+		version, err := parseVersion(reports[0].Version)
+		if err != nil {
+			http.Error(w, err.Error(), 400)
+			return
+		}
+		for _, r := range reports {
+			pkt := packet(version, "failure")
+			pkt.Message = r.Description
+			pkt.Extra = raven.Extra{
+				"count": r.Count,
+			}
+			for k, v := range r.Extra {
+				pkt.Extra[k] = v
+			}
+			if r.Goroutines != "" {
+				url, err := saveFailureWithGoroutines(r.FailureData, failureDir)
+				if err != nil {
+					log.Println("Saving failure report:", err)
+					http.Error(w, "Internal server error", http.StatusInternalServerError)
+					return
+				}
+				pkt.Extra["goroutinesURL"] = url
+			}
+			message := sanitizeMessageLDB(r.Description)
+			pkt.Fingerprint = []string{message}
+
+			if err := sendReport(dsn, pkt, userIDFor(req)); err != nil {
+				log.Println("Failed to send failure report:", err)
+			} else {
+				log.Println("Sent failure report:", r.Description)
+			}
+		}
+	}
+}
+
+func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string, error) {
+	bs := make([]byte, len(data.Description)+len(data.Goroutines))
+	copy(bs, data.Description)
+	copy(bs[len(data.Description):], data.Goroutines)
+	id := fmt.Sprintf("%x", sha256.Sum256(bs))
+	path := fullPathCompressed(failureDir, id)
+	err := compressAndWrite(bs, path)
+	if err != nil {
+		return "", err
+	}
+	return reportServer + path, nil
+}

cmd/stcrashreceiver/sentry.go

@@ -8,14 +8,16 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"errors"
-	"io/ioutil"
+	"io"
+	"log"
 	"regexp"
 	"strings"
 	"sync"
 
 	raven "github.com/getsentry/raven-go"
-	"github.com/maruel/panicparse/stack"
+	"github.com/maruel/panicparse/v2/stack"
 )
 
 const reportServer = "https://crash.syncthing.net/report/"
@@ -31,12 +33,45 @@ var (
 	clientsMut sync.Mutex
 )
 
-func sendReport(dsn, path string, report []byte, userID string) error {
+type sentryService struct {
-	pkt, err := parseReport(path, report)
+	dsn   string
-	if err != nil {
+	inbox chan sentryRequest
-		return err
+}
-	}
 
+type sentryRequest struct {
+	reportID string
+	data     []byte
+}
+
+func (s *sentryService) Serve(ctx context.Context) {
+	for {
+		select {
+		case req := <-s.inbox:
+			pkt, err := parseCrashReport(req.reportID, req.data)
+			if err != nil {
+				log.Println("Failed to parse crash report:", err)
+				continue
+			}
+			if err := sendReport(s.dsn, pkt, req.reportID); err != nil {
+				log.Println("Failed to send crash report:", err)
+			}
+
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (s *sentryService) Send(reportID string, data []byte) bool {
+	select {
+	case s.inbox <- sentryRequest{reportID, data}:
+		return true
+	default:
+		return false
+	}
+}
+
+func sendReport(dsn string, pkt *raven.Packet, userID string) error {
 	pkt.Interfaces = append(pkt.Interfaces, &raven.User{ID: userID})
 
 	clientsMut.Lock()
@@ -44,6 +79,7 @@ func sendReport(dsn, path string, report []byte, userID string) error {
 
 	cli, ok := clients[dsn]
 	if !ok {
+		var err error
 		cli, err = raven.New(dsn)
 		if err != nil {
 			return err
@@ -62,7 +98,7 @@ func sendReport(dsn, path string, report []byte, userID string) error {
 	return <-errC
 }
 
-func parseReport(path string, report []byte) (*raven.Packet, error) {
+func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
 	parts := bytes.SplitN(report, []byte("\n"), 2)
 	if len(parts) != 2 {
 		return nil, errors.New("no first line")
@@ -97,10 +133,13 @@ func parseReport(path string, report []byte) (*raven.Packet, error) {
 	}
 
 	r := bytes.NewReader(report)
-	ctx, err := stack.ParseDump(r, ioutil.Discard, false)
+	ctx, _, err := stack.ScanSnapshot(r, io.Discard, stack.DefaultOpts())
-	if err != nil {
+	if err != nil && err != io.EOF {
 		return nil, err
 	}
+	if ctx == nil || len(ctx.Goroutines) == 0 {
+		return nil, errors.New("no goroutines found")
+	}
 
 	// Lock the source code loader to the version we are processing here.
 	if version.commit != "" {
@@ -111,7 +150,7 @@ func parseReport(path string, report []byte) (*raven.Packet, error) {
 		loader.LockWithVersion(version.tag)
 	} else {
 		// Last resort
-		loader.LockWithVersion("master")
+		loader.LockWithVersion("main")
 	}
 	defer loader.Unlock()
 
@@ -120,50 +159,74 @@ func parseReport(path string, report []byte) (*raven.Packet, error) {
 		if gr.First {
 			trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
 			for i, sc := range gr.Stack.Calls {
-				trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name(), sc.SrcPath, sc.Line, 3, nil)
+				trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name, sc.RemoteSrcPath, sc.Line, 3, nil)
 			}
 			break
 		}
 	}
 
-	pkt := &raven.Packet{
+	pkt := packet(version, "crash")
-		Message:     string(subjectLine),
+	pkt.Message = string(subjectLine)
-		Platform:    "go",
+	pkt.Extra = raven.Extra{
-		Release:     version.tag,
+		"url": reportServer + path,
-		Environment: version.environment(),
-		Tags: raven.Tags{
-			raven.Tag{Key: "version", Value: version.version},
-			raven.Tag{Key: "tag", Value: version.tag},
-			raven.Tag{Key: "codename", Value: version.codename},
-			raven.Tag{Key: "runtime", Value: version.runtime},
-			raven.Tag{Key: "goos", Value: version.goos},
-			raven.Tag{Key: "goarch", Value: version.goarch},
-			raven.Tag{Key: "builder", Value: version.builder},
-		},
-		Extra: raven.Extra{
-			"url": reportServer + path,
-		},
-		Interfaces: []raven.Interface{&trace},
-	}
-	if version.commit != "" {
-		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
 	}
+	pkt.Interfaces = []raven.Interface{&trace}
+	pkt.Fingerprint = crashReportFingerprint(pkt.Message)
 
 	return pkt, nil
 }
 
-// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC
+var (
-var longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)`)
+	indexRe          = regexp.MustCompile(`\[[-:0-9]+\]`)
+	sizeRe           = regexp.MustCompile(`(length|capacity) [0-9]+`)
+	ldbPosRe         = regexp.MustCompile(`(\(pos=)([0-9]+)\)`)
+	ldbChecksumRe    = regexp.MustCompile(`(want=0x)([a-z0-9]+)( got=0x)([a-z0-9]+)`)
+	ldbFileRe        = regexp.MustCompile(`(\[file=)([0-9]+)(\.ldb\])`)
+	ldbInternalKeyRe = regexp.MustCompile(`(internal key ")[^"]+(", len=)[0-9]+`)
+	ldbPathRe        = regexp.MustCompile(`(open|write|read) .+[\\/].+[\\/]index[^\\/]+[\\/][^\\/]+: `)
+)
+
+func sanitizeMessageLDB(message string) string {
+	message = ldbPosRe.ReplaceAllString(message, "${1}x)")
+	message = ldbFileRe.ReplaceAllString(message, "${1}x${3}")
+	message = ldbChecksumRe.ReplaceAllString(message, "${1}X${3}X")
+	message = ldbInternalKeyRe.ReplaceAllString(message, "${1}x${2}x")
+	message = ldbPathRe.ReplaceAllString(message, "$1 x: ")
+	return message
+}
+
+func crashReportFingerprint(message string) []string {
+	// Do not fingerprint on the stack in case of db corruption or fatal
+	// db io error - where it occurs doesn't matter.
+	orig := message
+	message = sanitizeMessageLDB(message)
+	if message != orig {
+		return []string{message}
+	}
+
+	message = indexRe.ReplaceAllString(message, "[x]")
+	message = sizeRe.ReplaceAllString(message, "$1 x")
+
+	// {{ default }} is what sentry uses as a fingerprint by default. While
+	// never specified, the docs point at this being some hash derived from the
+	// stack trace. Here we include the filtered panic message on top of that.
+	// https://docs.sentry.io/platforms/go/data-management/event-grouping/sdk-fingerprinting/#basic-example
+	return []string{"{{ default }}", message}
+}
+
+// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]
+var longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
 
 type version struct {
-	version  string // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
+	version  string   // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
-	tag      string // "v1.1.4-rc.1"
+	tag      string   // "v1.1.4-rc.1"
-	commit   string // "6aaae618", blank when absent
+	commit   string   // "6aaae618", blank when absent
-	codename string // "Erbium Earthworm"
+	codename string   // "Erbium Earthworm"
-	runtime  string // "go1.12.5"
+	runtime  string   // "go1.12.5"
-	goos     string // "darwin"
+	goos     string   // "darwin"
-	goarch   string // "amd64"
+	goarch   string   // "amd64"
-	builder  string // "jb@kvin.kastelo.net"
+	builder  string   // "jb@kvin.kastelo.net"
+	extra    []string // "foo", "bar"
 }
 
 func (v version) environment() string {
@@ -193,6 +256,7 @@ func parseVersion(line string) (version, error) {
 		goarch:   m[5],
 		builder:  m[6],
 	}
+
 	parts := strings.Split(v.version, "+")
 	v.tag = parts[0]
 	if len(parts) > 1 {
@@ -202,5 +266,38 @@ func parseVersion(line string) (version, error) {
 		}
 	}
 
+	if len(m) >= 8 && m[7] != "" {
+		tags := strings.Split(m[7], ",")
+		for i := range tags {
+			tags[i] = strings.TrimSpace(tags[i])
+		}
+		v.extra = tags
+	}
+
 	return v, nil
 }
+
+func packet(version version, reportType string) *raven.Packet {
+	pkt := &raven.Packet{
+		Platform:    "go",
+		Release:     version.tag,
+		Environment: version.environment(),
+		Tags: raven.Tags{
+			raven.Tag{Key: "version", Value: version.version},
+			raven.Tag{Key: "tag", Value: version.tag},
+			raven.Tag{Key: "codename", Value: version.codename},
+			raven.Tag{Key: "runtime", Value: version.runtime},
+			raven.Tag{Key: "goos", Value: version.goos},
+			raven.Tag{Key: "goarch", Value: version.goarch},
+			raven.Tag{Key: "builder", Value: version.builder},
+			raven.Tag{Key: "report_type", Value: reportType},
+		},
+	}
+	if version.commit != "" {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
+	}
+	for _, tag := range version.extra {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: tag, Value: "1"})
+	}
+	return pkt
+}

cmd/stcrashreceiver/sentry_test.go

@@ -8,7 +8,7 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
+	"os"
 	"testing"
 )
 
@@ -30,27 +30,41 @@ func TestParseVersion(t *testing.T) {
 				builder:  "jb@kvin.kastelo.net",
 			},
 		},
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+				extra:    []string{"foo", "bar"},
+			},
+		},
 	}
 
 	for _, tc := range cases {
 		v, err := parseVersion(tc.longVersion)
 		if err != nil {
-			t.Error(err)
+			t.Errorf("%s\nerror: %v\n", tc.longVersion, err)
 			continue
 		}
-		if v != tc.parsed {
+		if fmt.Sprint(v) != fmt.Sprint(tc.parsed) {
-			t.Error(v)
+			t.Errorf("%s\nA: %v\nE: %v\n", tc.longVersion, v, tc.parsed)
 		}
 	}
 }
 
 func TestParseReport(t *testing.T) {
-	bs, err := ioutil.ReadFile("_testdata/panic.log")
+	bs, err := os.ReadFile("_testdata/panic.log")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	pkt, err := parseReport("1/2/345", bs)
+	pkt, err := parseCrashReport("1/2/345", bs)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -62,3 +76,66 @@ func TestParseReport(t *testing.T) {
 
 	fmt.Printf("%s\n", bs)
 }
+
+func TestCrashReportFingerprint(t *testing.T) {
+	cases := []struct {
+		message, exp string
+		ldb          bool
+	}{
+		{
+			message: "panic: leveldb/table: corruption on data-block (pos=51308946): checksum mismatch, want=0xa89f9aa0 got=0xd27cc4c7 [file=004003.ldb]",
+			exp:     "panic: leveldb/table: corruption on data-block (pos=x): checksum mismatch, want=0xX got=0xX [file=x.ldb]",
+			ldb:     true,
+		},
+		{
+			message: "panic: leveldb/table: corruption on table-footer (pos=248): bad magic number [file=001370.ldb]",
+			exp:     "panic: leveldb/table: corruption on table-footer (pos=x): bad magic number [file=x.ldb]",
+			ldb:     true,
+		},
+		{
+			message: "panic: runtime error: slice bounds out of range [4294967283:4194304]",
+			exp:     "panic: runtime error: slice bounds out of range [x]",
+		},
+		{
+			message: "panic: runtime error: slice bounds out of range [-2:]",
+			exp:     "panic: runtime error: slice bounds out of range [x]",
+		},
+		{
+			message: "panic: runtime error: slice bounds out of range [:4294967283] with capacity 32768",
+			exp:     "panic: runtime error: slice bounds out of range [x] with capacity x",
+		},
+		{
+			message: "panic: runtime error: index out of range [0] with length 0",
+			exp:     "panic: runtime error: index out of range [x] with length x",
+		},
+		{
+			message: `panic: leveldb: internal key "\x01", len=1: invalid length`,
+			exp:     `panic: leveldb: internal key "x", len=x: invalid length`,
+			ldb:     true,
+		},
+		{
+			message: `panic: write /var/syncthing/config/index-v0.14.0.db/2732813.log: cannot allocate memory`,
+			exp:     `panic: write x: cannot allocate memory`,
+			ldb:     true,
+		},
+		{
+			message: `panic: filling Blocks: read C:\Users\Serv-Resp-Tizayuca\AppData\Local\Syncthing\index-v0.14.0.db\006561.ldb: Error de datos (comprobación de redundancia cíclica).`,
+			exp:     `panic: filling Blocks: read x: Error de datos (comprobación de redundancia cíclica).`,
+			ldb:     true,
+		},
+	}
+
+	for i, tc := range cases {
+		fingerprint := crashReportFingerprint(tc.message)
+
+		expLen := 2
+		if tc.ldb {
+			expLen = 1
+		}
+		if l := len(fingerprint); l != expLen {
+			t.Errorf("tc %v: Unexpected fingerprint length: %v != %v", i, l, expLen)
+		} else if msg := fingerprint[expLen-1]; msg != tc.exp {
+			t.Errorf("tc %v:\n\"%v\" !=\n\"%v\"", i, msg, tc.exp)
+		}
+	}
+}

cmd/stcrashreceiver/sourcecodeloader.go

@@ -9,7 +9,7 @@ package main
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"path/filepath"
 	"strings"
@@ -72,11 +72,15 @@ func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]b
 		url := urlPrefix + l.version + filename[idx:]
 		resp, err := l.client.Get(url)
 
-		if err != nil || resp.StatusCode != http.StatusOK {
+		if err != nil {
-			fmt.Println("Loading source:", err.Error())
+			fmt.Println("Loading source:", err)
 			return nil, 0
 		}
-		data, err := ioutil.ReadAll(resp.Body)
+		if resp.StatusCode != http.StatusOK {
+			fmt.Println("Loading source:", resp.Status)
+			return nil, 0
+		}
+		data, err := io.ReadAll(resp.Body)
 		_ = resp.Body.Close()
 		if err != nil {
 			fmt.Println("Loading source:", err.Error())

cmd/stcrashreceiver/stcrashreceiver.go

@@ -4,55 +4,19 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// Command stcrashreceiver is a trivial HTTP server that allows two things:
-//
-// - uploading files (crash reports) named like a SHA256 hash using a PUT request
-// - checking whether such file exists using a HEAD request
-//
-// Typically this should be deployed behind something that manages HTTPS.
 package main
 
 import (
-	"bytes"
-	"compress/gzip"
-	"flag"
-	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
-	"net"
 	"net/http"
-	"os"
 	"path"
-	"path/filepath"
 	"strings"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/sha256"
 )
 
-const maxRequestSize = 1 << 20 // 1 MiB
-
-func main() {
-	dir := flag.String("dir", ".", "Directory to store reports in")
-	dsn := flag.String("dsn", "", "Sentry DSN")
-	listen := flag.String("listen", ":22039", "HTTP listen address")
-	flag.Parse()
-
-	cr := &crashReceiver{
-		dir: *dir,
-		dsn: *dsn,
-	}
-
-	log.SetOutput(os.Stdout)
-	if err := http.ListenAndServe(*listen, cr); err != nil {
-		log.Fatalln("HTTP serve:", err)
-	}
-}
-
 type crashReceiver struct {
-	dir string
+	store  *diskStore
-	dsn string
+	sentry *sentryService
 }
 
 func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
@@ -75,111 +39,55 @@ func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	// The location of the report on disk, compressed
-	fullPath := filepath.Join(r.dir, r.dirFor(reportID), reportID) + ".gz"
-
 	switch req.Method {
 	case http.MethodGet:
-		r.serveGet(fullPath, w, req)
+		r.serveGet(reportID, w, req)
 	case http.MethodHead:
-		r.serveHead(fullPath, w, req)
+		r.serveHead(reportID, w, req)
 	case http.MethodPut:
-		r.servePut(reportID, fullPath, w, req)
+		r.servePut(reportID, w, req)
 	default:
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 	}
 }
 
 // serveGet responds to GET requests by serving the uncompressed report.
-func (r *crashReceiver) serveGet(fullPath string, w http.ResponseWriter, _ *http.Request) {
+func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http.Request) {
-	fd, err := os.Open(fullPath)
+	bs, err := r.store.Get(reportID)
 	if err != nil {
 		http.Error(w, "Not found", http.StatusNotFound)
 		return
 	}
-
+	w.Write(bs)
-	defer fd.Close()
-	gr, err := gzip.NewReader(fd)
-	if err != nil {
-		http.Error(w, "Internal server error", http.StatusInternalServerError)
-		return
-	}
-	_, _ = io.Copy(w, gr) // best effort
 }
 
 // serveHead responds to HEAD requests by checking if the named report
 // already exists in the system.
-func (r *crashReceiver) serveHead(fullPath string, w http.ResponseWriter, _ *http.Request) {
+func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) {
-	if _, err := os.Lstat(fullPath); err != nil {
+	if !r.store.Exists(reportID) {
 		http.Error(w, "Not found", http.StatusNotFound)
 	}
 }
 
 // servePut accepts and stores the given report.
-func (r *crashReceiver) servePut(reportID, fullPath string, w http.ResponseWriter, req *http.Request) {
+func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *http.Request) {
-	// Ensure the destination directory exists
-	if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
-		log.Println("Creating directory:", err)
-		http.Error(w, "Internal server error", http.StatusInternalServerError)
-		return
-	}
-
 	// Read at most maxRequestSize of report data.
 	log.Println("Receiving report", reportID)
 	lr := io.LimitReader(req.Body, maxRequestSize)
-	bs, err := ioutil.ReadAll(lr)
+	bs, err := io.ReadAll(lr)
 	if err != nil {
 		log.Println("Reading report:", err)
 		http.Error(w, "Internal server error", http.StatusInternalServerError)
 		return
 	}
 
-	// Compress the report for storage
+	// Store the report
-	buf := new(bytes.Buffer)
+	if !r.store.Put(reportID, bs) {
-	gw := gzip.NewWriter(buf)
+		log.Println("Failed to store report (queue full):", reportID)
-	_, _ = gw.Write(bs) // can't fail
-	gw.Close()
-
-	// Create an output file with the compressed report
-	err = ioutil.WriteFile(fullPath, buf.Bytes(), 0644)
-	if err != nil {
-		log.Println("Saving report:", err)
-		http.Error(w, "Internal server error", http.StatusInternalServerError)
-		return
 	}
 
 	// Send the report to Sentry
-	if r.dsn != "" {
+	if !r.sentry.Send(reportID, bs) {
-		// Remote ID
+		log.Println("Failed to send report to sentry (queue full):", reportID)
-		user := userIDFor(req)
-
-		go func() {
-			// There's no need for the client to have to wait for this part.
-			if err := sendReport(r.dsn, reportID, bs, user); err != nil {
-				log.Println("Failed to send report:", err)
-			}
-		}()
 	}
 }
-
-// 01234567890abcdef... => 01/23
-func (r *crashReceiver) dirFor(base string) string {
-	return filepath.Join(base[0:2], base[2:4])
-}
-
-// userIDFor returns a string we can use as the user ID for the purpose of
-// counting affected users. It's the truncated hash of a salt, the user
-// remote IP, and the current month.
-func userIDFor(req *http.Request) string {
-	addr := req.RemoteAddr
-	if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
-		addr = fwd
-	}
-	if host, _, err := net.SplitHostPort(addr); err == nil {
-		addr = host
-	}
-	now := time.Now().Format("200601")
-	salt := "stcrashreporter"
-	hash := sha256.Sum256([]byte(salt + addr + now))
-	return fmt.Sprintf("%x", hash[:8])
-}

cmd/stcrashreceiver/util.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/sha256"
+)
+
+// userIDFor returns a string we can use as the user ID for the purpose of
+// counting affected users. It's the truncated hash of a salt, the user
+// remote IP, and the current month.
+func userIDFor(req *http.Request) string {
+	addr := req.RemoteAddr
+	if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
+		addr = fwd
+	}
+	if host, _, err := net.SplitHostPort(addr); err == nil {
+		addr = host
+	}
+	now := time.Now().Format("200601")
+	salt := "stcrashreporter"
+	hash := sha256.Sum256([]byte(salt + addr + now))
+	return fmt.Sprintf("%x", hash[:8])
+}
+
+// 01234567890abcdef... => 01/23
+func dirFor(base string) string {
+	return filepath.Join(base[0:2], base[2:4])
+}
+
+func fullPathCompressed(root, reportID string) string {
+	return filepath.Join(root, dirFor(reportID), reportID) + ".gz"
+}
+
+func compressAndWrite(bs []byte, fullPath string) error {
+	// Compress the report for storage
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	_, _ = gw.Write(bs) // can't fail
+	gw.Close()
+
+	// Create an output file with the compressed report
+	return os.WriteFile(fullPath, buf.Bytes(), 0644)
+}

cmd/stdisco/main.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/binary"
 	"flag"
@@ -47,14 +48,16 @@ func main() {
 		log.Println("My ID:", myID)
 	}
 
-	runbeacon(beacon.NewMulticast(mc), fake)
+	ctx := context.Background()
-	runbeacon(beacon.NewBroadcast(bc), fake)
+
+	runbeacon(ctx, beacon.NewMulticast(mc), fake)
+	runbeacon(ctx, beacon.NewBroadcast(bc), fake)
 
 	select {}
 }
 
-func runbeacon(bc beacon.Interface, fake bool) {
+func runbeacon(ctx context.Context, bc beacon.Interface, fake bool) {
-	go bc.Serve()
+	go bc.Serve(ctx)
 	go recv(bc)
 	if fake {
 		go send(bc)

cmd/stdiscosrv/apisrv.go

@@ -10,8 +10,10 @@ import (
 	"bytes"
 	"context"
 	"crypto/tls"
+	"encoding/base64"
 	"encoding/json"
 	"encoding/pem"
+	"errors"
 	"fmt"
 	"log"
 	"math/rand"
@@ -66,12 +68,12 @@ func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator,
 	}
 }
 
-func (s *apiSrv) Serve() {
+func (s *apiSrv) Serve(_ context.Context) error {
 	if s.useHTTP {
 		listener, err := net.Listen("tcp", s.addr)
 		if err != nil {
 			log.Println("Listen:", err)
-			return
+			return err
 		}
 		s.listener = listener
 	} else {
@@ -93,7 +95,7 @@ func (s *apiSrv) Serve() {
 		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
 		if err != nil {
 			log.Println("Listen:", err)
-			return
+			return err
 		}
 		s.listener = tlsListener
 	}
@@ -107,9 +109,11 @@ func (s *apiSrv) Serve() {
 		MaxHeaderBytes: httpMaxHeaderBytes,
 	}
 
-	if err := srv.Serve(s.listener); err != nil {
+	err := srv.Serve(s.listener)
+	if err != nil {
 		log.Println("Serve:", err)
 	}
+	return err
 }
 
 var topCtx = context.Background()
@@ -132,11 +136,19 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 		log.Println(reqID, req.Method, req.URL)
 	}
 
-	var remoteIP net.IP
+	remoteAddr := &net.TCPAddr{
+		IP:   nil,
+		Port: -1,
+	}
+
 	if s.useHTTP {
-		remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
+		remoteAddr.IP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
+		if parsedPort, err := strconv.ParseInt(req.Header.Get("X-Client-Port"), 10, 0); err == nil {
+			remoteAddr.Port = int(parsedPort)
+		}
 	} else {
-		addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
+		var err error
+		remoteAddr, err = net.ResolveTCPAddr("tcp", req.RemoteAddr)
 		if err != nil {
 			log.Println("remoteAddr:", err)
 			lw.Header().Set("Retry-After", errorRetryAfterString())
@@ -144,14 +156,13 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 			apiRequestsTotal.WithLabelValues("no_remote_addr").Inc()
 			return
 		}
-		remoteIP = addr.IP
 	}
 
 	switch req.Method {
 	case "GET":
 		s.handleGET(ctx, lw, req)
 	case "POST":
-		s.handlePOST(ctx, remoteIP, lw, req)
+		s.handlePOST(ctx, remoteAddr, lw, req)
 	default:
 		http.Error(lw, "Method Not Allowed", http.StatusMethodNotAllowed)
 	}
@@ -217,13 +228,13 @@ func (s *apiSrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http
 	w.Write(bs)
 }
 
-func (s *apiSrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
+func (s *apiSrv) handlePOST(ctx context.Context, remoteAddr *net.TCPAddr, w http.ResponseWriter, req *http.Request) {
 	reqID := ctx.Value(idKey).(requestID)
 
-	rawCert := certificateBytes(req)
+	rawCert, err := certificateBytes(req)
-	if rawCert == nil {
+	if err != nil {
 		if debug {
-			log.Println(reqID, "no certificates")
+			log.Println(reqID, "no certificates:", err)
 		}
 		announceRequestsTotal.WithLabelValues("no_certificate").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
@@ -244,7 +255,7 @@ func (s *apiSrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.Respons
 
 	deviceID := protocol.NewDeviceID(rawCert)
 
-	addresses := fixupAddresses(remoteIP, ann.Addresses)
+	addresses := fixupAddresses(remoteAddr, ann.Addresses)
 	if len(addresses) == 0 {
 		announceRequestsTotal.WithLabelValues("bad_request").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
@@ -252,7 +263,7 @@ func (s *apiSrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.Respons
 		return
 	}
 
-	if err := s.handleAnnounce(remoteIP, deviceID, addresses); err != nil {
+	if err := s.handleAnnounce(deviceID, addresses); err != nil {
 		announceRequestsTotal.WithLabelValues("internal_error").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
@@ -269,7 +280,7 @@ func (s *apiSrv) Stop() {
 	s.listener.Close()
 }
 
-func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addresses []string) error {
+func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string) error {
 	key := deviceID.String()
 	now := time.Now()
 	expire := now.Add(addressExpiryTime).UnixNano()
@@ -291,13 +302,13 @@ func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addre
 	return s.db.merge(key, dbAddrs, seen)
 }
 
-func handlePing(w http.ResponseWriter, r *http.Request) {
+func handlePing(w http.ResponseWriter, _ *http.Request) {
 	w.WriteHeader(204)
 }
 
-func certificateBytes(req *http.Request) []byte {
+func certificateBytes(req *http.Request) ([]byte, error) {
 	if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
-		return req.TLS.PeerCertificates[0].Raw
+		return req.TLS.PeerCertificates[0].Raw, nil
 	}
 
 	var bs []byte
@@ -310,7 +321,7 @@ func certificateBytes(req *http.Request) []byte {
 			hdr, err := url.QueryUnescape(hdr)
 			if err != nil {
 				// Decoding failed
-				return nil
+				return nil, err
 			}
 
 			bs = []byte(hdr)
@@ -329,6 +340,15 @@ func certificateBytes(req *http.Request) []byte {
 				}
 			}
 		}
+	} else if hdr := req.Header.Get("X-Tls-Client-Cert-Der-Base64"); hdr != "" {
+		// Caddy {tls_client_certificate_der_base64}
+		hdr, err := base64.StdEncoding.DecodeString(hdr)
+		if err != nil {
+			// Decoding failed
+			return nil, err
+		}
+
+		bs = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: hdr})
 	} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
 		// Traefik 2 passtlsclientcert
 		// The certificate is in PEM format with url encoding but without newlines
@@ -337,7 +357,7 @@ func certificateBytes(req *http.Request) []byte {
 		hdr, err := url.QueryUnescape(hdr)
 		if err != nil {
 			// Decoding failed
-			return nil
+			return nil, err
 		}
 
 		for i := 64; i < len(hdr); i += 65 {
@@ -350,21 +370,21 @@ func certificateBytes(req *http.Request) []byte {
 	}
 
 	if bs == nil {
-		return nil
+		return nil, errors.New("empty certificate header")
 	}
 
 	block, _ := pem.Decode(bs)
 	if block == nil {
 		// Decoding failed
-		return nil
+		return nil, errors.New("certificate decode result is empty")
 	}
 
-	return block.Bytes
+	return block.Bytes, nil
 }
 
 // fixupAddresses checks the list of addresses, removing invalid ones and
 // replacing unspecified IPs with the given remote IP.
-func fixupAddresses(remote net.IP, addresses []string) []string {
+func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
 	fixed := make([]string, 0, len(addresses))
 	for _, annAddr := range addresses {
 		uri, err := url.Parse(annAddr)
@@ -384,27 +404,34 @@ func fixupAddresses(remote net.IP, addresses []string) []string {
 			continue
 		}
 
-		if host == "" || ip.IsUnspecified() {
+		if remote != nil {
-			// Replace the unspecified IP with the request source.
+			if host == "" || ip.IsUnspecified() {
+				// Replace the unspecified IP with the request source.
 
-			// ... unless the request source is the loopback address or
+				// ... unless the request source is the loopback address or
-			// multicast/unspecified (can't happen, really).
+				// multicast/unspecified (can't happen, really).
-			if remote.IsLoopback() || remote.IsMulticast() || remote.IsUnspecified() {
+				if remote.IP.IsLoopback() || remote.IP.IsMulticast() || remote.IP.IsUnspecified() {
-				continue
+					continue
+				}
+
+				// Do not use IPv6 remote address if requested scheme is ...4
+				// (i.e., tcp4, etc.)
+				if strings.HasSuffix(uri.Scheme, "4") && remote.IP.To4() == nil {
+					continue
+				}
+
+				// Do not use IPv4 remote address if requested scheme is ...6
+				if strings.HasSuffix(uri.Scheme, "6") && remote.IP.To4() != nil {
+					continue
+				}
+
+				host = remote.IP.String()
 			}
 
-			// Do not use IPv6 remote address if requested scheme is ...4
+			// If zero port was specified, use remote port.
-			// (i.e., tcp4, etc.)
+			if port == "0" && remote.Port > 0 {
-			if strings.HasSuffix(uri.Scheme, "4") && remote.To4() == nil {
+				port = strconv.Itoa(remote.Port)
-				continue
 			}
-
-			// Do not use IPv4 remote address if requested scheme is ...6
-			if strings.HasSuffix(uri.Scheme, "6") && remote.To4() != nil {
-				continue
-			}
-
-			host = remote.String()
 		}
 
 		uri.Host = net.JoinHostPort(host, port)

cmd/stdiscosrv/apisrv_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestFixupAddresses(t *testing.T) {
 	cases := []struct {
-		remote net.IP
+		remote *net.TCPAddr
 		in     []string
 		out    []string
 	}{
@@ -22,37 +22,53 @@ func TestFixupAddresses(t *testing.T) {
 			in:  []string{"tcp://1.2.3.4:22000"},
 			out: []string{"tcp://1.2.3.4:22000"},
 		}, { // unspecified replaced by remote
-			remote: net.ParseIP("1.2.3.4"),
+			remote: addr("1.2.3.4", 22000),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://1.2.3.4:22000", "tcp://192.0.2.42:22000"},
 		}, { // unspecified not used as replacement
-			remote: net.ParseIP("0.0.0.0"),
+			remote: addr("0.0.0.0", 22000),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // unspecified not used as replacement
-			remote: net.ParseIP("::"),
+			remote: addr("::", 22000),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // localhost not used as replacement
-			remote: net.ParseIP("127.0.0.1"),
+			remote: addr("127.0.0.1", 22000),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // localhost not used as replacement
-			remote: net.ParseIP("::1"),
+			remote: addr("::1", 22000),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // multicast not used as replacement
-			remote: net.ParseIP("224.0.0.1"),
+			remote: addr("224.0.0.1", 22000),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // multicast not used as replacement
-			remote: net.ParseIP("ff80::42"),
+			remote: addr("ff80::42", 22000),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // explicitly announced weirdness is also filtered
-			remote: net.ParseIP("192.0.2.42"),
+			remote: addr("192.0.2.42", 22000),
 			in:     []string{"tcp://:22000", "tcp://127.1.2.3:22000", "tcp://[::1]:22000", "tcp://[ff80::42]:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
+		}, { // port remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://0.0.0.0:0"},
+			out:    []string{"tcp://123.123.123.123:9000"},
+		}, { // unspecified port remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://:0"},
+			out:    []string{"tcp://123.123.123.123:9000"},
+		}, { // empty remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://"},
+			out:    []string{},
+		}, { // port only remapping
+			remote: addr("123.123.123.123", 9000),
+			in:     []string{"tcp://44.44.44.44:0"},
+			out:    []string{"tcp://44.44.44.44:9000"},
 		},
 	}
 
@@ -63,3 +79,10 @@ func TestFixupAddresses(t *testing.T) {
 		}
 	}
 }
+
+func addr(host string, port int) *net.TCPAddr {
+	return &net.TCPAddr{
+		IP:   net.ParseIP(host),
+		Port: port,
+	}
+}

cmd/stdiscosrv/database.go

@@ -4,17 +4,19 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//go:generate go run ../../script/protofmt.go database.proto
+//go:generate go run ../../proto/scripts/protofmt.go database.proto
 //go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
 
 package main
 
 import (
+	"context"
 	"log"
 	"sort"
 	"time"
 
 	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
@@ -37,7 +39,6 @@ type database interface {
 type levelDBStore struct {
 	db         *leveldb.DB
 	inbox      chan func()
-	stop       chan struct{}
 	clock      clock
 	marshalBuf []byte
 }
@@ -50,7 +51,18 @@ func newLevelDBStore(dir string) (*levelDBStore, error) {
 	return &levelDBStore{
 		db:    db,
 		inbox: make(chan func(), 16),
-		stop:  make(chan struct{}),
+		clock: defaultClock{},
+	}, nil
+}
+
+func newMemoryLevelDBStore() (*levelDBStore, error) {
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		return nil, err
+	}
+	return &levelDBStore{
+		db:    db,
+		inbox: make(chan func(), 16),
 		clock: defaultClock{},
 	}, nil
 }
@@ -155,7 +167,7 @@ func (s *levelDBStore) get(key string) (DatabaseRecord, error) {
 	return rec, nil
 }
 
-func (s *levelDBStore) Serve() {
+func (s *levelDBStore) Serve(ctx context.Context) error {
 	t := time.NewTimer(0)
 	defer t.Stop()
 	defer s.db.Close()
@@ -183,7 +195,7 @@ loop:
 			// the next.
 			t.Reset(databaseStatisticsInterval)
 
-		case <-s.stop:
+		case <-ctx.Done():
 			// We're done.
 			close(statisticsTrigger)
 			break loop
@@ -192,6 +204,8 @@ loop:
 
 	// Also wait for statisticsServe to return
 	<-statisticsDone
+
+	return nil
 }
 
 func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
@@ -255,10 +269,6 @@ func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- stru
 	}
 }
 
-func (s *levelDBStore) Stop() {
-	close(s.stop)
-}
-
 // merge returns the merged result of the two database records a and b. The
 // result is the union of the two address sets, with the newer expiry time
 // chosen for any duplicates.

cmd/stdiscosrv/database.pb.go

@@ -510,10 +510,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if (iNdEx + skippy) < 0 {
 				return ErrInvalidLengthDatabase
 			}
 			if (iNdEx + skippy) > l {
@@ -648,10 +645,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if (iNdEx + skippy) < 0 {
 				return ErrInvalidLengthDatabase
 			}
 			if (iNdEx + skippy) > l {
@@ -752,10 +746,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if (iNdEx + skippy) < 0 {
 				return ErrInvalidLengthDatabase
 			}
 			if (iNdEx + skippy) > l {

cmd/stdiscosrv/database_test.go

@@ -7,21 +7,20 @@
 package main
 
 import (
+	"context"
 	"fmt"
-	"os"
 	"testing"
 	"time"
 )
 
 func TestDatabaseGetSet(t *testing.T) {
-	os.RemoveAll("_database")
+	db, err := newMemoryLevelDBStore()
-	defer os.RemoveAll("_database")
-	db, err := newLevelDBStore("_database")
 	if err != nil {
 		t.Fatal(err)
 	}
-	go db.Serve()
+	ctx, cancel := context.WithCancel(context.Background())
-	defer db.Stop()
+	go db.Serve(ctx)
+	defer cancel()
 
 	// Check missing record
 
@@ -117,7 +116,7 @@ func TestDatabaseGetSet(t *testing.T) {
 
 	// Put a record with misses
 
-	rec = DatabaseRecord{Misses: 42}
+	rec = DatabaseRecord{Misses: 42, Missed: tc.Now().UnixNano()}
 	if err := db.put("efgh", rec); err != nil {
 		t.Fatal(err)
 	}
@@ -207,5 +206,6 @@ func (t *testClock) wind(d time.Duration) {
 }
 
 func (t *testClock) Now() time.Time {
+	t.now = t.now.Add(time.Nanosecond)
 	return t.now
 }

cmd/stdiscosrv/main.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
 	"log"
@@ -21,7 +22,7 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/thejerf/suture"
+	"github.com/thejerf/suture/v4"
 )
 
 const (
@@ -115,8 +116,11 @@ func main() {
 	var replicationDestinations []string
 	parts := strings.Split(replicationPeers, ",")
 	for _, part := range parts {
-		fields := strings.Split(part, "@")
+		if part == "" {
+			continue
+		}
 
+		fields := strings.Split(part, "@")
 		switch len(fields) {
 		case 2:
 			// This is an id@address specification. Grab the address for the
@@ -136,6 +140,9 @@ func main() {
 			if err != nil {
 				log.Fatalln("Parsing device ID:", err)
 			}
+			if id == protocol.EmptyDeviceID {
+				log.Fatalf("Missing device ID for peer in %q", part)
+			}
 			allowedReplicationPeers = append(allowedReplicationPeers, id)
 
 		default:
@@ -183,5 +190,5 @@ func main() {
 	}
 
 	// Engage!
-	main.Serve()
+	main.Serve(context.Background())
 }

cmd/stdiscosrv/replication.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"encoding/binary"
 	"fmt"
@@ -32,7 +33,6 @@ type replicationSender struct {
 	cert       tls.Certificate // our certificate
 	allowedIDs []protocol.DeviceID
 	outbox     chan ReplicationRecord
-	stop       chan struct{}
 }
 
 func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
@@ -41,11 +41,10 @@ func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protoco
 		cert:       cert,
 		allowedIDs: allowedIDs,
 		outbox:     make(chan ReplicationRecord, replicationOutboxSize),
-		stop:       make(chan struct{}),
 	}
 }
 
-func (s *replicationSender) Serve() {
+func (s *replicationSender) Serve(ctx context.Context) error {
 	// Sleep a little at startup. Peers often restart at the same time, and
 	// this avoid the service failing and entering backoff state
 	// unnecessarily, while also reducing the reconnect rate to something
@@ -62,7 +61,7 @@ func (s *replicationSender) Serve() {
 	conn, err := tls.Dial("tcp", s.dst, tlsCfg)
 	if err != nil {
 		log.Println("Replication connect:", err)
-		return
+		return err
 	}
 	defer func() {
 		conn.SetWriteDeadline(time.Now().Add(time.Second))
@@ -73,13 +72,13 @@ func (s *replicationSender) Serve() {
 	remoteID, err := deviceID(conn)
 	if err != nil {
 		log.Println("Replication connect:", err)
-		return
+		return err
 	}
 
 	// Verify it's in the set of allowed device IDs.
 	if !deviceIDIn(remoteID, s.allowedIDs) {
 		log.Println("Replication connect: unexpected device ID:", remoteID)
-		return
+		return err
 	}
 
 	heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
@@ -121,26 +120,22 @@ func (s *replicationSender) Serve() {
 			if _, err := conn.Write(buf[:4+n]); err != nil {
 				replicationSendsTotal.WithLabelValues("error").Inc()
 				log.Println("Replication write:", err)
-				// Yes, we are loosing the replication event here.
+				// Yes, we are losing the replication event here.
-				return
+				return err
 			}
 			replicationSendsTotal.WithLabelValues("success").Inc()
 
-		case <-s.stop:
+		case <-ctx.Done():
-			return
+			return nil
 		}
 	}
 }
 
-func (s *replicationSender) Stop() {
-	close(s.stop)
-}
-
 func (s *replicationSender) String() string {
 	return fmt.Sprintf("replicationSender(%q)", s.dst)
 }
 
-func (s *replicationSender) send(key string, ps []DatabaseAddress, seen int64) {
+func (s *replicationSender) send(key string, ps []DatabaseAddress, _ int64) {
 	item := ReplicationRecord{
 		Key:       key,
 		Addresses: ps,
@@ -172,7 +167,6 @@ type replicationListener struct {
 	cert       tls.Certificate
 	allowedIDs []protocol.DeviceID
 	db         database
-	stop       chan struct{}
 }
 
 func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
@@ -181,11 +175,10 @@ func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []prot
 		cert:       cert,
 		allowedIDs: allowedIDs,
 		db:         db,
-		stop:       make(chan struct{}),
 	}
 }
 
-func (l *replicationListener) Serve() {
+func (l *replicationListener) Serve(ctx context.Context) error {
 	tlsCfg := &tls.Config{
 		Certificates:       []tls.Certificate{l.cert},
 		ClientAuth:         tls.RequestClientCert,
@@ -196,14 +189,14 @@ func (l *replicationListener) Serve() {
 	lst, err := tls.Listen("tcp", l.addr, tlsCfg)
 	if err != nil {
 		log.Println("Replication listen:", err)
-		return
+		return err
 	}
 	defer lst.Close()
 
 	for {
 		select {
-		case <-l.stop:
+		case <-ctx.Done():
-			return
+			return nil
 		default:
 		}
 
@@ -211,7 +204,7 @@ func (l *replicationListener) Serve() {
 		conn, err := lst.Accept()
 		if err != nil {
 			log.Println("Replication accept:", err)
-			return
+			return err
 		}
 
 		// Figure out the other side device ID
@@ -231,19 +224,15 @@ func (l *replicationListener) Serve() {
 			continue
 		}
 
-		go l.handle(conn)
+		go l.handle(ctx, conn)
 	}
 }
 
-func (l *replicationListener) Stop() {
-	close(l.stop)
-}
-
 func (l *replicationListener) String() string {
 	return fmt.Sprintf("replicationListener(%q)", l.addr)
 }
 
-func (l *replicationListener) handle(conn net.Conn) {
+func (l *replicationListener) handle(ctx context.Context, conn net.Conn) {
 	defer func() {
 		conn.SetWriteDeadline(time.Now().Add(time.Second))
 		conn.Close()
@@ -253,7 +242,7 @@ func (l *replicationListener) handle(conn net.Conn) {
 
 	for {
 		select {
-		case <-l.stop:
+		case <-ctx.Done():
 			return
 		default:
 		}

cmd/stdiscosrv/stats.go

@@ -10,6 +10,7 @@ import (
 	"os"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
 )
 
 var (
@@ -109,7 +110,7 @@ func init() {
 		databaseKeys, databaseStatisticsSeconds,
 		databaseOperations, databaseOperationSeconds)
 
-	processCollectorOpts := prometheus.ProcessCollectorOpts{
+	processCollectorOpts := collectors.ProcessCollectorOpts{
 		Namespace: "syncthing_discovery",
 		PidFn: func() (int, error) {
 			return os.Getpid(), nil
@@ -117,7 +118,7 @@ func init() {
 	}
 
 	prometheus.MustRegister(
-		prometheus.NewProcessCollector(processCollectorOpts),
+		collectors.NewProcessCollector(processCollectorOpts),
 	)
 
 }

cmd/stevents/main.go

@@ -28,16 +28,21 @@ func main() {
 	log.SetFlags(0)
 
 	target := flag.String("target", "localhost:8384", "Target Syncthing instance")
+	types := flag.String("types", "", "Filter for specific event types (comma-separated)")
 	apikey := flag.String("apikey", "", "Syncthing API key")
 	flag.Parse()
 
 	if *apikey == "" {
 		log.Fatal("Must give -apikey argument")
 	}
+	var eventsArg string
+	if len(*types) > 0 {
+		eventsArg = "&events=" + *types
+	}
 
 	since := 0
 	for {
-		req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d", *target, since), nil)
+		req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d%s", *target, since, eventsArg), nil)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/stfinddevice/main.go

@@ -84,7 +84,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 }
 
 func checkServer(deviceID protocol.DeviceID, server string) checkResult {
-	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger)
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger, nil)
 	if err != nil {
 		return checkResult{error: err}
 	}

cmd/stfindignored/main.go

@@ -4,7 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// Commmand stfindignored lists ignored files under a given folder root.
+// Command stfindignored lists ignored files under a given folder root.
 package main
 
 import (

cmd/stindex/dump.go

@@ -1,112 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"encoding/binary"
-	"fmt"
-	"log"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/db/backend"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-func dump(ldb backend.Backend) {
-	it, err := ldb.NewPrefixIterator(nil)
-	if err != nil {
-		log.Fatal(err)
-	}
-	for it.Next() {
-		key := it.Key()
-		switch key[0] {
-		case db.KeyTypeDevice:
-			folder := binary.BigEndian.Uint32(key[1:])
-			device := binary.BigEndian.Uint32(key[1+4:])
-			name := nulString(key[1+4+4:])
-			fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
-
-			var f protocol.FileInfo
-			err := f.Unmarshal(it.Value())
-			if err != nil {
-				log.Fatal(err)
-			}
-			fmt.Printf(" V:%v\n", f)
-
-		case db.KeyTypeGlobal:
-			folder := binary.BigEndian.Uint32(key[1:])
-			name := nulString(key[1+4:])
-			var flv db.VersionList
-			flv.Unmarshal(it.Value())
-			fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
-
-		case db.KeyTypeBlock:
-			folder := binary.BigEndian.Uint32(key[1:])
-			hash := key[1+4 : 1+4+32]
-			name := nulString(key[1+4+32:])
-			fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
-
-		case db.KeyTypeDeviceStatistic:
-			fmt.Printf("[dstat] K:%x V:%x\n", it.Key(), it.Value())
-
-		case db.KeyTypeFolderStatistic:
-			fmt.Printf("[fstat] K:%x V:%x\n", it.Key(), it.Value())
-
-		case db.KeyTypeVirtualMtime:
-			folder := binary.BigEndian.Uint32(key[1:])
-			name := nulString(key[1+4:])
-			val := it.Value()
-			var real, virt time.Time
-			real.UnmarshalBinary(val[:len(val)/2])
-			virt.UnmarshalBinary(val[len(val)/2:])
-			fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, real, virt)
-
-		case db.KeyTypeFolderIdx:
-			key := binary.BigEndian.Uint32(it.Key()[1:])
-			fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
-
-		case db.KeyTypeDeviceIdx:
-			key := binary.BigEndian.Uint32(it.Key()[1:])
-			val := it.Value()
-			if len(val) == 0 {
-				fmt.Printf("[deviceidx] K:%d V:<nil>\n", key)
-			} else {
-				dev := protocol.DeviceIDFromBytes(val)
-				fmt.Printf("[deviceidx] K:%d V:%s\n", key, dev)
-			}
-
-		case db.KeyTypeIndexID:
-			device := binary.BigEndian.Uint32(it.Key()[1:])
-			folder := binary.BigEndian.Uint32(it.Key()[5:])
-			fmt.Printf("[indexid] D:%d F:%d I:%x\n", device, folder, it.Value())
-
-		case db.KeyTypeFolderMeta:
-			folder := binary.BigEndian.Uint32(it.Key()[1:])
-			fmt.Printf("[foldermeta] F:%d V:%x\n", folder, it.Value())
-
-		case db.KeyTypeMiscData:
-			fmt.Printf("[miscdata] K:%q V:%q\n", it.Key()[1:], it.Value())
-
-		case db.KeyTypeSequence:
-			folder := binary.BigEndian.Uint32(it.Key()[1:])
-			seq := binary.BigEndian.Uint64(it.Key()[5:])
-			fmt.Printf("[sequence] F:%d S:%d V:%q\n", folder, seq, it.Value())
-
-		case db.KeyTypeNeed:
-			folder := binary.BigEndian.Uint32(it.Key()[1:])
-			file := string(it.Key()[5:])
-			fmt.Printf("[need] F:%d V:%q\n", folder, file)
-
-		case db.KeyTypeBlockList:
-			fmt.Printf("[blocklist] H:%x\n", it.Key()[1:])
-
-		default:
-			fmt.Printf("[???]\n  %x\n  %x\n", it.Key(), it.Value())
-		}
-	}
-}

cmd/stindex/main.go

@@ -1,52 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-
-	"github.com/syncthing/syncthing/lib/db/backend"
-)
-
-func main() {
-	var mode string
-	log.SetFlags(0)
-	log.SetOutput(os.Stdout)
-
-	flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize, idxck")
-
-	flag.Parse()
-
-	path := flag.Arg(0)
-	if path == "" {
-		path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
-	}
-
-	ldb, err := backend.OpenLevelDBRO(path)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	switch mode {
-	case "dump":
-		dump(ldb)
-	case "dumpsize":
-		dumpsize(ldb)
-	case "idxck":
-		if !idxck(ldb) {
-			os.Exit(1)
-		}
-	case "account":
-		account(ldb)
-	default:
-		fmt.Println("Unknown mode")
-	}
-}

cmd/stindex/util.go

@@ -1,52 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"log"
-	"os"
-	"path/filepath"
-	"runtime"
-
-	"github.com/syncthing/syncthing/lib/fs"
-)
-
-func nulString(bs []byte) string {
-	for i := range bs {
-		if bs[i] == 0 {
-			return string(bs[:i])
-		}
-	}
-	return string(bs)
-}
-
-func defaultConfigDir() string {
-	switch runtime.GOOS {
-	case "windows":
-		if p := os.Getenv("LocalAppData"); p != "" {
-			return filepath.Join(p, "Syncthing")
-		}
-		return filepath.Join(os.Getenv("AppData"), "Syncthing")
-
-	case "darwin":
-		dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
-		if err != nil {
-			log.Fatal(err)
-		}
-		return dir
-
-	default:
-		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
-			return filepath.Join(xdgCfg, "syncthing")
-		}
-		dir, err := fs.ExpandTilde("~/.config/syncthing")
-		if err != nil {
-			log.Fatal(err)
-		}
-		return dir
-	}
-}

cmd/strelaypoolsrv/auto/noassets.go

@@ -0,0 +1,16 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//go:build noassets
+// +build noassets
+
+package auto
+
+import "github.com/syncthing/syncthing/lib/assets"
+
+func Assets() map[string]assets.Asset {
+	return nil
+}

cmd/strelaypoolsrv/gui/index.html

@@ -46,16 +46,18 @@
       <h1>Relay Pool Data</h1>
       <div ng-if="relays === undefined" class="text-center">
         <img src="https://cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
-        <p>Please wait while we gather data</p>
+        <p>Please wait while we gather data…</p>
       </div>
       <div>
         <div ng-show="relays !== undefined" class="ng-hide">
           <p>
-            Currently {{ relays.length }} relays online ({{ totals.goMaxProcs }} cores in total).
+            The relays listed on this page are not managed or vetted by the Syncthing project.
+            Each relay is the responsibility of the relay operator.
+            Currently {{ relays.length }} relays are online.
           </p>
         </div>
         <div id="map"></div> <!-- Can't hide the map, otherwise it freaks out -->
-        <p>The circle size represents how much bytes the relay transferred relative to other relays</p>
+        <p>The circle size represents how much bytes the relay has transferred relatively to other relays.</p>
       </div>
       <div>
         <table class="table table-striped table-condensed table">
@@ -186,7 +188,7 @@
       <hr>
       <p>
         This product includes GeoLite2 data created by MaxMind, available from
-        <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
+        <a href="https://www.maxmind.com">https://www.maxmind.com</a>.
       </p>
     </div>
 
@@ -235,7 +237,7 @@
       uptimeSeconds: 0,
     };
     $scope.map = L.map('map').setView([40.90296, 1.90925], 2);
-    L.tileLayer('http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',
+    L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',
     {
       attribution: 'Leaflet',
       maxZoom: 17

cmd/strelaypoolsrv/main.go

@@ -3,14 +3,12 @@
 package main
 
 import (
-	"compress/gzip"
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"flag"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
@@ -19,9 +17,13 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
-	"github.com/golang/groupcache/lru"
+	lru "github.com/hashicorp/golang-lru/v2"
+	"github.com/syncthing/syncthing/lib/httpcache"
+	"github.com/syncthing/syncthing/lib/protocol"
+
 	"github.com/oschwald/geoip2-golang"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -31,7 +33,6 @@ import (
 	"github.com/syncthing/syncthing/lib/relay/client"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/tlsutil"
-	"golang.org/x/time/rate"
 )
 
 type location struct {
@@ -97,27 +98,13 @@ var (
 	dir               string
 	evictionTime      = time.Hour
 	debug             bool
-	getLRUSize        = 10 << 10
-	getLimitBurst     = 10
-	getLimitAvg       = 2
-	postLRUSize       = 1 << 10
-	postLimitBurst    = 2
-	postLimitAvg      = 2
-	getLimit          time.Duration
-	postLimit         time.Duration
 	permRelaysFile    string
 	ipHeader          string
 	geoipPath         string
 	proto             string
-	statsRefresh      = time.Minute / 2
+	statsRefresh      = time.Minute
-	requestQueueLen   = 10
+	requestQueueLen   = 64
-	requestProcessors = 1
+	requestProcessors = 8
-
-	getMut      = sync.NewMutex()
-	getLRUCache *lru.Cache
-
-	postMut      = sync.NewMutex()
-	postLRUCache *lru.Cache
 
 	requests chan request
 
@@ -125,6 +112,7 @@ var (
 	knownRelays     = make([]*relay, 0)
 	permanentRelays = make([]*relay, 0)
 	evictionTimers  = make(map[string]*time.Timer)
+	globalBlocklist = newErrorTracker(1000)
 )
 
 const (
@@ -139,13 +127,8 @@ func main() {
 	flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
 	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
 	flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
-	flag.IntVar(&getLRUSize, "get-limit-cache", getLRUSize, "Get request limiter cache size")
-	flag.IntVar(&getLimitAvg, "get-limit-avg", getLimitAvg, "Allowed average get request rate, per 10 s")
-	flag.IntVar(&getLimitBurst, "get-limit-burst", getLimitBurst, "Allowed burst get requests")
-	flag.IntVar(&postLRUSize, "post-limit-cache", postLRUSize, "Post request limiter cache size")
-	flag.IntVar(&postLimitAvg, "post-limit-avg", postLimitAvg, "Allowed average post request rate, per minute")
-	flag.IntVar(&postLimitBurst, "post-limit-burst", postLimitBurst, "Allowed burst post requests")
 	flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
+	flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays")
 	flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
 	flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
 	flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
@@ -157,12 +140,6 @@ func main() {
 
 	requests = make(chan request, requestQueueLen)
 
-	getLimit = 10 * time.Second / time.Duration(getLimitAvg)
-	postLimit = time.Minute / time.Duration(postLimitAvg)
-
-	getLRUCache = lru.New(getLRUSize)
-	postLRUCache = lru.New(postLRUSize)
-
 	var listener net.Listener
 	var err error
 
@@ -208,6 +185,7 @@ func main() {
 		tlsCfg := &tls.Config{
 			Certificates: []tls.Certificate{cert},
 			MinVersion:   tls.VersionTLS10, // No SSLv3
+			ClientAuth:   tls.RequestClientCert,
 			CipherSuites: []uint16{
 				// No RC4
 				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
@@ -237,7 +215,7 @@ func main() {
 
 	handler := http.NewServeMux()
 	handler.HandleFunc("/", handleAssets)
-	handler.HandleFunc("/endpoint", handleRequest)
+	handler.Handle("/endpoint", httpcache.SinglePath(http.HandlerFunc(handleRequest), 15*time.Second))
 	handler.HandleFunc("/metrics", handleMetrics)
 
 	srv := http.Server{
@@ -268,17 +246,13 @@ func handleAssets(w http.ResponseWriter, r *http.Request) {
 		path = "index.html"
 	}
 
-	content, ok := auto.Assets()[path]
+	as, ok := auto.Assets()[path]
 	if !ok {
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
 
-	assets.Serve(w, r, assets.Asset{
+	assets.Serve(w, r, as)
-		ContentGz: content,
-		Filename:  path,
-		Modified:  time.Unix(auto.Generated, 0).UTC(),
-	})
 }
 
 func handleRequest(w http.ResponseWriter, r *http.Request) {
@@ -292,21 +266,17 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 	}()
 
 	if ipHeader != "" {
-		r.RemoteAddr = r.Header.Get(ipHeader)
+		hdr := r.Header.Get(ipHeader)
+		fields := strings.Split(hdr, ",")
+		if len(fields) > 0 {
+			r.RemoteAddr = strings.TrimSpace(fields[len(fields)-1])
+		}
 	}
 	w.Header().Set("Access-Control-Allow-Origin", "*")
 	switch r.Method {
 	case "GET":
-		if limit(r.RemoteAddr, getLRUCache, getMut, getLimit, getLimitBurst) {
-			w.WriteHeader(httpStatusEnhanceYourCalm)
-			return
-		}
 		handleGetRequest(w, r)
 	case "POST":
-		if limit(r.RemoteAddr, postLRUCache, postMut, postLimit, postLimitBurst) {
-			w.WriteHeader(httpStatusEnhanceYourCalm)
-			return
-		}
 		handlePostRequest(w, r)
 	default:
 		if debug {
@@ -318,27 +288,44 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 
 func handleGetRequest(rw http.ResponseWriter, r *http.Request) {
 	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
+
 	mut.RLock()
-	relays := append(permanentRelays, knownRelays...)
+	relays := make([]*relay, len(permanentRelays)+len(knownRelays))
+	n := copy(relays, permanentRelays)
+	copy(relays[n:], knownRelays)
 	mut.RUnlock()
 
 	// Shuffle
 	rand.Shuffle(relays)
 
-	w := io.Writer(rw)
+	_ = json.NewEncoder(rw).Encode(map[string][]*relay{
-	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
-		rw.Header().Set("Content-Encoding", "gzip")
-		gw := gzip.NewWriter(rw)
-		defer gw.Close()
-		w = gw
-	}
-
-	_ = json.NewEncoder(w).Encode(map[string][]*relay{
 		"relays": relays,
 	})
 }
 
 func handlePostRequest(w http.ResponseWriter, r *http.Request) {
+	// Get the IP address of the client
+	rhost := r.RemoteAddr
+	if host, _, err := net.SplitHostPort(rhost); err == nil {
+		rhost = host
+	}
+
+	// Check the black list. A client is blacklisted if their last 10
+	// attempts to join have all failed. The "Unauthorized" status return
+	// causes strelaysrv to cease attempting to join.
+	if globalBlocklist.IsBlocked(rhost) {
+		log.Println("Rejected blocked client", rhost)
+		http.Error(w, "Too many errors", http.StatusUnauthorized)
+		globalBlocklist.ClearErrors(rhost)
+		return
+	}
+
+	var relayCert *x509.Certificate
+	if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
+		relayCert = r.TLS.PeerCertificates[0]
+		log.Printf("Got TLS cert from relay server")
+	}
+
 	var newRelay relay
 	err := json.NewDecoder(r.Body).Decode(&newRelay)
 	r.Body.Close()
@@ -360,6 +347,21 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Canonicalize the URL. In particular, parse and re-encode the query
+	// string so that it's guaranteed to be valid.
+	uri.RawQuery = uri.Query().Encode()
+	newRelay.URL = uri.String()
+
+	if relayCert != nil {
+		advertisedId := uri.Query().Get("id")
+		idFromCert := protocol.NewDeviceID(relayCert.Raw).String()
+		if advertisedId != idFromCert {
+			log.Println("Warning: Relay server requested to join with an ID different from the join request, rejecting")
+			http.Error(w, "mismatched advertised id and join request cert", http.StatusBadRequest)
+			return
+		}
+	}
+
 	host, port, err := net.SplitHostPort(uri.Host)
 	if err != nil {
 		if debug {
@@ -369,18 +371,12 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Get the IP address of the client
-	rhost := r.RemoteAddr
-	if host, _, err := net.SplitHostPort(rhost); err == nil {
-		rhost = host
-	}
-
 	ip := net.ParseIP(host)
 	// The client did not provide an IP address, use the IP address of the client.
 	if ip == nil || ip.IsUnspecified() {
 		uri.Host = net.JoinHostPort(rhost, port)
 		newRelay.URL = uri.String()
-	} else if host != rhost {
+	} else if host != rhost && relayCert == nil {
 		if debug {
 			log.Println("IP address advertised does not match client IP address", r.RemoteAddr, uri)
 		}
@@ -406,10 +402,14 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 	case requests <- request{&newRelay, reschan, prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("queue"))}:
 		result := <-reschan
 		if result.err != nil {
+			log.Println("Join from", r.RemoteAddr, "failed:", result.err)
+			globalBlocklist.AddError(rhost)
 			relayTestsTotal.WithLabelValues("failed").Inc()
 			http.Error(w, result.err.Error(), http.StatusBadRequest)
 			return
 		}
+		log.Println("Join from", r.RemoteAddr, "succeeded")
+		globalBlocklist.ClearErrors(rhost)
 		relayTestsTotal.WithLabelValues("success").Inc()
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		json.NewEncoder(w).Encode(map[string]time.Duration{
@@ -457,7 +457,7 @@ func handleRelayTest(request request) {
 		updateMetrics(request.relay.uri.Host, *stats, location)
 	}
 	request.relay.Stats = stats
-	request.relay.StatsRetrieved = time.Now()
+	request.relay.StatsRetrieved = time.Now().Truncate(time.Second)
 	request.relay.Location = location
 
 	timer, ok := evictionTimers[request.relay.uri.Host]
@@ -523,25 +523,8 @@ func evict(relay *relay) func() {
 	}
 }
 
-func limit(addr string, cache *lru.Cache, lock sync.Mutex, intv time.Duration, burst int) bool {
-	if host, _, err := net.SplitHostPort(addr); err == nil {
-		addr = host
-	}
-
-	lock.Lock()
-	v, _ := cache.Get(addr)
-	bkt, ok := v.(*rate.Limiter)
-	if !ok {
-		bkt = rate.NewLimiter(rate.Every(intv), burst)
-		cache.Add(addr, bkt)
-	}
-	lock.Unlock()
-
-	return !bkt.Allow()
-}
-
 func loadRelays(file string) []*relay {
-	content, err := ioutil.ReadFile(file)
+	content, err := os.ReadFile(file)
 	if err != nil {
 		log.Println("Failed to load relays: " + err.Error())
 		return nil
@@ -549,7 +532,7 @@ func loadRelays(file string) []*relay {
 
 	var relays []*relay
 	for _, line := range strings.Split(string(content), "\n") {
-		if len(line) == 0 {
+		if line == "" {
 			continue
 		}
 
@@ -579,11 +562,11 @@ func saveRelays(file string, relays []*relay) error {
 	for _, relay := range relays {
 		content += relay.uri.String() + "\n"
 	}
-	return ioutil.WriteFile(file, []byte(content), 0777)
+	return os.WriteFile(file, []byte(content), 0o777)
 }
 
 func createTestCertificate() tls.Certificate {
-	tmpDir, err := ioutil.TempDir("", "relaypoolsrv")
+	tmpDir, err := os.MkdirTemp("", "relaypoolsrv")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -638,3 +621,42 @@ func (lrw *loggingResponseWriter) WriteHeader(code int) {
 	lrw.statusCode = code
 	lrw.ResponseWriter.WriteHeader(code)
 }
+
+type errorTracker struct {
+	errors *lru.TwoQueueCache[string, *errorCounter]
+}
+
+type errorCounter struct {
+	count atomic.Int32
+}
+
+func newErrorTracker(size int) *errorTracker {
+	cache, err := lru.New2Q[string, *errorCounter](size)
+	if err != nil {
+		panic(err)
+	}
+	return &errorTracker{
+		errors: cache,
+	}
+}
+
+func (b *errorTracker) AddError(host string) {
+	entry, ok := b.errors.Get(host)
+	if !ok {
+		entry = &errorCounter{}
+		b.errors.Add(host, entry)
+	}
+	c := entry.count.Add(1)
+	log.Printf("Error count for %s is now %d", host, c)
+}
+
+func (b *errorTracker) ClearErrors(host string) {
+	b.errors.Remove(host)
+}
+
+func (b *errorTracker) IsBlocked(host string) bool {
+	if be, ok := b.errors.Get(host); ok {
+		return be.count.Load() > 10
+	}
+	return false
+}

cmd/strelaypoolsrv/main_test.go

@@ -0,0 +1,94 @@
+// Copyright © 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"sync"
+	"testing"
+)
+
+func init() {
+	for i := 0; i < 10; i++ {
+		u := fmt.Sprintf("permanent%d", i)
+		permanentRelays = append(permanentRelays, &relay{URL: u})
+	}
+
+	knownRelays = []*relay{
+		{URL: "known1"},
+		{URL: "known2"},
+		{URL: "known3"},
+	}
+
+	mut = new(sync.RWMutex)
+}
+
+// Regression test: handleGetRequest should not modify permanentRelays.
+func TestHandleGetRequest(t *testing.T) {
+	needcap := len(permanentRelays) + len(knownRelays)
+	if needcap > cap(permanentRelays) {
+		t.Fatalf("test setup failed: need cap(permanentRelays) >= %d, have %d",
+			needcap, cap(permanentRelays))
+	}
+
+	w := httptest.NewRecorder()
+	w.Body = new(bytes.Buffer)
+	handleGetRequest(w, httptest.NewRequest("GET", "/", nil))
+
+	result := make(map[string][]*relay)
+	err := json.NewDecoder(w.Body).Decode(&result)
+	if err != nil {
+		t.Fatalf("invalid JSON: %v", err)
+	}
+
+	relays := result["relays"]
+	expect, actual := len(knownRelays)+len(permanentRelays), len(relays)
+	if actual != expect {
+		t.Errorf("expected %d relays, got %d", expect, actual)
+	}
+
+	// Check for changes in permanentRelays.
+	for i, r := range permanentRelays {
+		switch {
+		case !strings.HasPrefix(r.URL, "permanent"):
+			t.Errorf("relay %q among permanent relays", r.URL)
+		case r.URL != fmt.Sprintf("permanent%d", i):
+			t.Error("order of permanent relays changed")
+		}
+	}
+}
+
+func TestCanonicalizeQueryValues(t *testing.T) {
+	// This just demonstrates and validates the uri.Parse/String stuff in
+	// regards to query strings.
+
+	in := "http://example.com/?some weird= query^value"
+	exp := "http://example.com/?some+weird=+query%5Evalue"
+
+	uri, err := url.Parse(in)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	str := uri.String()
+	if str != in {
+		// Just re-encoding the URL doesn't sanitize the query string.
+		t.Errorf("expected %q, got %q", in, str)
+	}
+
+	uri.RawQuery = uri.Query().Encode()
+	str = uri.String()
+	if str != exp {
+		// The query string is now in correct format.
+		t.Errorf("expected %q, got %q", exp, str)
+	}
+}

cmd/strelaypoolsrv/stats.go

@@ -10,11 +10,12 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
 func init() {
-	processCollectorOpts := prometheus.ProcessCollectorOpts{
+	processCollectorOpts := collectors.ProcessCollectorOpts{
 		Namespace: "syncthing_relaypoolsrv",
 		PidFn: func() (int, error) {
 			return os.Getpid(), nil
@@ -22,7 +23,7 @@ func init() {
 	}
 
 	prometheus.MustRegister(
-		prometheus.NewProcessCollector(processCollectorOpts),
+		collectors.NewProcessCollector(processCollectorOpts),
 	)
 }
 

cmd/strelaysrv/README.md

@@ -121,11 +121,11 @@ Relay related libraries used by this repo
 ----
 ##### Relay protocol definition.
 
-[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/protocol)
+[Available here](https://github.com/syncthing/syncthing/tree/main/lib/relay/protocol)
 
 
 ##### Relay client
 
 Only used by the testutil.
 
-[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/client)
+[Available here](https://github.com/syncthing/syncthing/tree/main/lib/relay/client)

cmd/strelaysrv/listener.go

@@ -20,10 +20,10 @@ import (
 var (
 	outboxesMut    = sync.RWMutex{}
 	outboxes       = make(map[syncthingprotocol.DeviceID]chan interface{})
-	numConnections int64
+	numConnections atomic.Int64
 )
 
-func listener(proto, addr string, config *tls.Config) {
+func listener(_, addr string, config *tls.Config, token string) {
 	tcpListener, err := net.Listen("tcp", addr)
 	if err != nil {
 		log.Fatalln(err)
@@ -49,7 +49,7 @@ func listener(proto, addr string, config *tls.Config) {
 		}
 
 		if isTLS {
-			go protocolConnectionHandler(conn, config)
+			go protocolConnectionHandler(conn, config, token)
 		} else {
 			go sessionConnectionHandler(conn)
 		}
@@ -57,7 +57,7 @@ func listener(proto, addr string, config *tls.Config) {
 	}
 }
 
-func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
+func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token string) {
 	conn := tls.Server(tcpConn, config)
 	if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil {
 		if debug {
@@ -76,7 +76,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
 	}
 
 	state := conn.ConnectionState()
-	if (!state.NegotiatedProtocolIsMutual || state.NegotiatedProtocol != protocol.ProtocolName) && debug {
+	if debug && state.NegotiatedProtocol != protocol.ProtocolName {
 		log.Println("Protocol negotiation error")
 	}
 
@@ -119,7 +119,16 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
 
 			switch msg := message.(type) {
 			case protocol.JoinRelayRequest:
-				if atomic.LoadInt32(&overLimit) > 0 {
+				if token != "" && msg.Token != token {
+					if debug {
+						log.Printf("invalid token %s\n", msg.Token)
+					}
+					protocol.WriteMessage(conn, protocol.ResponseWrongToken)
+					conn.Close()
+					continue
+				}
+
+				if overLimit.Load() {
 					protocol.WriteMessage(conn, protocol.RelayFull{})
 					if debug {
 						log.Println("Refusing join request from", id, "due to being over limits")
@@ -149,7 +158,15 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
 				protocol.WriteMessage(conn, protocol.ResponseSuccess)
 
 			case protocol.ConnectRequest:
-				requestedPeer := syncthingprotocol.DeviceIDFromBytes(msg.ID)
+				requestedPeer, err := syncthingprotocol.DeviceIDFromBytes(msg.ID)
+				if err != nil {
+					if debug {
+						log.Println(id, "is looking for an invalid peer ID")
+					}
+					protocol.WriteMessage(conn, protocol.ResponseNotFound)
+					conn.Close()
+					continue
+				}
 				outboxesMut.RLock()
 				peerOutbox, ok := outboxes[requestedPeer]
 				outboxesMut.RUnlock()
@@ -250,7 +267,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
 				conn.Close()
 			}
 
-			if atomic.LoadInt32(&overLimit) > 0 && !hasSessions(id) {
+			if overLimit.Load() && !hasSessions(id) {
 				if debug {
 					log.Println("Dropping", id, "as it has no sessions and we are over our limits")
 				}
@@ -343,8 +360,8 @@ func sessionConnectionHandler(conn net.Conn) {
 }
 
 func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) {
-	atomic.AddInt64(&numConnections, 1)
+	numConnections.Add(1)
-	defer atomic.AddInt64(&numConnections, -1)
+	defer numConnections.Add(-1)
 
 	for {
 		msg, err := protocol.ReadMessage(conn)

cmd/strelaysrv/main.go

@@ -3,6 +3,7 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
 	"fmt"
@@ -13,7 +14,6 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"sync/atomic"
 	"syscall"
@@ -49,13 +49,14 @@ var (
 
 	sessionLimitBps   int
 	globalLimitBps    int
-	overLimit         int32
+	overLimit         atomic.Bool
 	descriptorLimit   int64
 	sessionLimiter    *rate.Limiter
 	globalLimiter     *rate.Limiter
 	networkBufferSize int
 
 	statusAddr       string
+	token            string
 	poolAddrs        string
 	pools            []string
 	providedBy       string
@@ -89,6 +90,7 @@ func main() {
 	flag.IntVar(&globalLimitBps, "global-rate", globalLimitBps, "Global rate limit, in bytes/s")
 	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
 	flag.StringVar(&statusAddr, "status-srv", ":22070", "Listen address for status service (blank to disable)")
+	flag.StringVar(&token, "token", "", "Token to restrict access to the relay (optional). Disables joining any pools.")
 	flag.StringVar(&poolAddrs, "pools", defaultPoolAddrs, "Comma separated list of relay pool addresses to join")
 	flag.StringVar(&providedBy, "provided-by", "", "An optional description about who provides the relay")
 	flag.StringVar(&extAddress, "ext-address", "", "An optional address to advertise as being available on.\n\tAllows listening on an unprivileged port with port forwarding from e.g. 443, and be connected to on port 443.")
@@ -98,7 +100,7 @@ func main() {
 	flag.IntVar(&natRenewal, "nat-renewal", 30, "NAT renewal frequency in minutes")
 	flag.IntVar(&natTimeout, "nat-timeout", 10, "NAT discovery timeout in seconds")
 	flag.BoolVar(&pprofEnabled, "pprof", false, "Enable the built in profiling on the status server")
-	flag.IntVar(&networkBufferSize, "network-buffer", 2048, "Network buffer size (two of these per proxied connection)")
+	flag.IntVar(&networkBufferSize, "network-buffer", 65536, "Network buffer size (two of these per proxied connection)")
 	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
@@ -145,7 +147,7 @@ func main() {
 		log.Println("Connection limit", descriptorLimit)
 
 		go monitorLimits()
-	} else if err != nil && runtime.GOOS != "windows" {
+	} else if err != nil && !build.IsWindows {
 		log.Println("Assuming no connection limit, due to error retrieving rlimits:", err)
 	}
 
@@ -184,19 +186,22 @@ func main() {
 		log.Println("ID:", id)
 	}
 
-	wrapper := config.Wrap("config", config.New(id), events.NoopLogger)
+	wrapper := config.Wrap("config", config.New(id), id, events.NoopLogger)
-	wrapper.SetOptions(config.OptionsConfiguration{
+	go wrapper.Serve(context.TODO())
-		NATLeaseM:   natLease,
+	wrapper.Modify(func(cfg *config.Configuration) {
-		NATRenewalM: natRenewal,
+		cfg.Options.NATLeaseM = natLease
-		NATTimeoutS: natTimeout,
+		cfg.Options.NATRenewalM = natRenewal
+		cfg.Options.NATTimeoutS = natTimeout
 	})
 	natSvc := nat.NewService(id, wrapper)
 	mapping := mapping{natSvc.NewMapping(nat.TCP, addr.IP, addr.Port)}
 
 	if natEnabled {
-		go natSvc.Serve()
+		ctx, cancel := context.WithCancel(context.Background())
+		go natSvc.Serve(ctx)
+		defer cancel()
 		found := make(chan struct{})
-		mapping.OnChanged(func(_ *nat.Mapping, _, _ []nat.Address) {
+		mapping.OnChanged(func() {
 			select {
 			case found <- struct{}{}:
 			default:
@@ -226,13 +231,37 @@ func main() {
 		go statusService(statusAddr)
 	}
 
-	uri, err := url.Parse(fmt.Sprintf("relay://%s/?id=%s&pingInterval=%s&networkTimeout=%s&sessionLimitBps=%d&globalLimitBps=%d&statusAddr=%s&providedBy=%s", mapping.Address(), id, pingInterval, networkTimeout, sessionLimitBps, globalLimitBps, statusAddr, providedBy))
+	uri, err := url.Parse(fmt.Sprintf("relay://%s/", mapping.Address()))
 	if err != nil {
 		log.Fatalln("Failed to construct URI", err)
+		return
 	}
 
+	// Add properly encoded query string parameters to URL.
+	query := make(url.Values)
+	query.Set("id", id.String())
+	query.Set("pingInterval", pingInterval.String())
+	query.Set("networkTimeout", networkTimeout.String())
+	if sessionLimitBps > 0 {
+		query.Set("sessionLimitBps", fmt.Sprint(sessionLimitBps))
+	}
+	if globalLimitBps > 0 {
+		query.Set("globalLimitBps", fmt.Sprint(globalLimitBps))
+	}
+	if statusAddr != "" {
+		query.Set("statusAddr", statusAddr)
+	}
+	if providedBy != "" {
+		query.Set("providedBy", providedBy)
+	}
+	uri.RawQuery = query.Encode()
+
 	log.Println("URI:", uri.String())
 
+	if token != "" {
+		poolAddrs = ""
+	}
+
 	if poolAddrs == defaultPoolAddrs {
 		log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
 		log.Println("!!  Joining default relay pools, this relay will be available for public use. !!")
@@ -244,11 +273,11 @@ func main() {
 	for _, pool := range pools {
 		pool = strings.TrimSpace(pool)
 		if len(pool) > 0 {
-			go poolHandler(pool, uri, mapping)
+			go poolHandler(pool, uri, mapping, cert)
 		}
 	}
 
-	go listener(proto, listen, tlsCfg)
+	go listener(proto, listen, tlsCfg, token)
 
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
@@ -279,10 +308,10 @@ func main() {
 func monitorLimits() {
 	limitCheckTimer = time.NewTimer(time.Minute)
 	for range limitCheckTimer.C {
-		if atomic.LoadInt64(&numConnections)+atomic.LoadInt64(&numProxies) > descriptorLimit {
+		if numConnections.Load()+numProxies.Load() > descriptorLimit {
-			atomic.StoreInt32(&overLimit, 1)
+			overLimit.Store(true)
 			log.Println("Gone past our connection limits. Starting to refuse new/drop idle connections.")
-		} else if atomic.CompareAndSwapInt32(&overLimit, 1, 0) {
+		} else if overLimit.CompareAndSwap(true, false) {
 			log.Println("Dropped below our connection limits. Accepting new connections.")
 		}
 		limitCheckTimer.Reset(time.Minute)

cmd/strelaysrv/pool.go

@@ -4,8 +4,9 @@ package main
 
 import (
 	"bytes"
+	"crypto/tls"
 	"encoding/json"
-	"io/ioutil"
+	"io"
 	"log"
 	"net/http"
 	"net/url"
@@ -16,7 +17,7 @@ const (
 	httpStatusEnhanceYourCalm = 429
 )
 
-func poolHandler(pool string, uri *url.URL, mapping mapping) {
+func poolHandler(pool string, uri *url.URL, mapping mapping, ownCert tls.Certificate) {
 	if debug {
 		log.Println("Joining", pool)
 	}
@@ -31,14 +32,31 @@ func poolHandler(pool string, uri *url.URL, mapping mapping) {
 			uriCopy.String(),
 		})
 
-		resp, err := httpClient.Post(pool, "application/json", &b)
+		poolUrl, err := url.Parse(pool)
+		if err != nil {
+			log.Printf("Could not parse pool url '%s': %v", pool, err)
+		}
+
+		client := http.DefaultClient
+		if poolUrl.Scheme == "https" {
+			// Sent our certificate in join request
+			client = &http.Client{
+				Transport: &http.Transport{
+					TLSClientConfig: &tls.Config{
+						Certificates: []tls.Certificate{ownCert},
+					},
+				},
+			}
+		}
+
+		resp, err := client.Post(pool, "application/json", &b)
 		if err != nil {
 			log.Printf("Error joining pool %v: HTTP request: %v", pool, err)
 			time.Sleep(time.Minute)
 			continue
 		}
 
-		bs, err := ioutil.ReadAll(resp.Body)
+		bs, err := io.ReadAll(resp.Body)
 		resp.Body.Close()
 		if err != nil {
 			log.Printf("Error joining pool %v: reading response: %v", pool, err)
@@ -51,14 +69,13 @@ func poolHandler(pool string, uri *url.URL, mapping mapping) {
 			var x struct {
 				EvictionIn time.Duration `json:"evictionIn"`
 			}
-			if err := json.NewDecoder(resp.Body).Decode(&x); err == nil {
+			if err := json.Unmarshal(bs, &x); err == nil {
 				rejoin := x.EvictionIn - (x.EvictionIn / 5)
 				log.Printf("Joined pool %s, rejoining in %v", pool, rejoin)
 				time.Sleep(rejoin)
 				continue
-			} else {
-				log.Printf("Joined pool %s, failed to deserialize response: %v", pool, err)
 			}
+			log.Printf("Joined pool %s, failed to deserialize response: %v", pool, err)
 
 		case http.StatusInternalServerError:
 			log.Printf("Failed to join %v: server error", pool)

cmd/strelaysrv/session.go

@@ -23,8 +23,8 @@ var (
 	sessionMut      = sync.RWMutex{}
 	activeSessions  = make([]*session, 0)
 	pendingSessions = make(map[string]*session)
-	numProxies      int64
+	numProxies      atomic.Int64
-	bytesProxied    int64
+	bytesProxied    atomic.Int64
 )
 
 func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionRateLimit, globalRateLimit *rate.Limiter) *session {
@@ -251,8 +251,8 @@ func (s *session) proxy(c1, c2 net.Conn) error {
 		log.Println("Proxy", c1.RemoteAddr(), "->", c2.RemoteAddr())
 	}
 
-	atomic.AddInt64(&numProxies, 1)
+	numProxies.Add(1)
-	defer atomic.AddInt64(&numProxies, -1)
+	defer numProxies.Add(-1)
 
 	buf := make([]byte, networkBufferSize)
 	for {
@@ -262,7 +262,7 @@ func (s *session) proxy(c1, c2 net.Conn) error {
 			return err
 		}
 
-		atomic.AddInt64(&bytesProxied, int64(n))
+		bytesProxied.Add(int64(n))
 
 		if debug {
 			log.Printf("%d bytes from %s to %s", n, c1.RemoteAddr(), c2.RemoteAddr())

cmd/strelaysrv/status.go

@@ -36,7 +36,7 @@ func statusService(addr string) {
 	}
 }
 
-func getStatus(w http.ResponseWriter, r *http.Request) {
+func getStatus(w http.ResponseWriter, _ *http.Request) {
 	w.Header().Set("Access-Control-Allow-Origin", "*")
 	status := make(map[string]interface{})
 
@@ -51,9 +51,9 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 	status["numPendingSessionKeys"] = len(pendingSessions)
 	status["numActiveSessions"] = len(activeSessions)
 	sessionMut.Unlock()
-	status["numConnections"] = atomic.LoadInt64(&numConnections)
+	status["numConnections"] = numConnections.Load()
-	status["numProxies"] = atomic.LoadInt64(&numProxies)
+	status["numProxies"] = numProxies.Load()
-	status["bytesProxied"] = atomic.LoadInt64(&bytesProxied)
+	status["bytesProxied"] = bytesProxied.Load()
 	status["goVersion"] = runtime.Version()
 	status["goOS"] = runtime.GOOS
 	status["goArch"] = runtime.GOARCH
@@ -88,13 +88,13 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 }
 
 type rateCalculator struct {
-	counter   *int64 // atomic, must remain 64-bit aligned
+	counter   *atomic.Int64
 	rates     []int64
 	prev      int64
 	startTime time.Time
 }
 
-func newRateCalculator(keepIntervals int, interval time.Duration, counter *int64) *rateCalculator {
+func newRateCalculator(keepIntervals int, interval time.Duration, counter *atomic.Int64) *rateCalculator {
 	r := &rateCalculator{
 		rates:     make([]int64, keepIntervals),
 		counter:   counter,
@@ -112,7 +112,7 @@ func (r *rateCalculator) updateRates(interval time.Duration) {
 		next := now.Truncate(interval).Add(interval)
 		time.Sleep(next.Sub(now))
 
-		cur := atomic.LoadInt64(r.counter)
+		cur := r.counter.Load()
 		rate := int64(float64(cur-r.prev) / interval.Seconds())
 		copy(r.rates[1:], r.rates)
 		r.rates[0] = rate

cmd/strelaysrv/testutil/main.go

@@ -20,7 +20,8 @@ import (
 )
 
 func main() {
-	ctx := context.Background()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
 	log.SetOutput(os.Stdout)
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
@@ -56,13 +57,13 @@ func main() {
 
 	if join {
 		log.Println("Creating client")
-		relay, err := client.NewClient(uri, []tls.Certificate{cert}, nil, 10*time.Second)
+		relay, err := client.NewClient(uri, []tls.Certificate{cert}, 10*time.Second)
 		if err != nil {
 			log.Fatal(err)
 		}
 		log.Println("Created client")
 
-		go relay.Serve()
+		go relay.Serve(ctx)
 
 		recv := make(chan protocol.SessionInvitation)
 
@@ -126,10 +127,6 @@ func stdinReader(c chan<- string) {
 }
 
 func connectToStdio(stdin <-chan string, conn net.Conn) {
-	go func() {
-
-	}()
-
 	buf := make([]byte, 1024)
 	for {
 		conn.SetReadDeadline(time.Now().Add(time.Millisecond))

cmd/stsigtool/main.go

@@ -9,7 +9,6 @@ package main
 import (
 	"flag"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 
@@ -69,7 +68,7 @@ func gen() {
 }
 
 func sign(keyname, dataname string) {
-	privkey, err := ioutil.ReadFile(keyname)
+	privkey, err := os.ReadFile(keyname)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -95,7 +94,7 @@ func sign(keyname, dataname string) {
 }
 
 func verifyWithFile(signame, dataname, keyname string) {
-	pubkey, err := ioutil.ReadFile(keyname)
+	pubkey, err := os.ReadFile(keyname)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -103,7 +102,7 @@ func verifyWithFile(signame, dataname, keyname string) {
 }
 
 func verifyWithKey(signame, dataname string, pubkey []byte) {
-	sig, err := ioutil.ReadFile(signame)
+	sig, err := os.ReadFile(signame)
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/stupgrades/main.go

@@ -7,31 +7,112 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
-	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
 	"os"
 	"sort"
+	"strings"
+	"time"
 
+	"github.com/alecthomas/kong"
+	"github.com/syncthing/syncthing/lib/httpcache"
 	"github.com/syncthing/syncthing/lib/upgrade"
 )
 
-const defaultURL = "https://api.github.com/repos/syncthing/syncthing/releases?per_page=25"
+type cli struct {
+	Listen    string        `default:":8080" help:"Listen address"`
+	URL       string        `short:"u" default:"https://api.github.com/repos/syncthing/syncthing/releases?per_page=25" help:"GitHub releases url"`
+	Forward   []string      `short:"f" help:"Forwarded pages, format: /path->https://example/com/url"`
+	CacheTime time.Duration `default:"15m" help:"Cache time"`
+}
 
 func main() {
-	url := flag.String("u", defaultURL, "GitHub releases url")
+	var params cli
-	flag.Parse()
+	kong.Parse(&params)
-
+	if err := server(&params); err != nil {
-	rels := upgrade.FetchLatestReleases(*url, "")
+		fmt.Printf("Error: %v\n", err)
-	if rels == nil {
-		// An error was already logged
 		os.Exit(1)
 	}
+}
+
+func server(params *cli) error {
+	http.Handle("/meta.json", httpcache.SinglePath(&githubReleases{url: params.URL}, params.CacheTime))
+
+	for _, fwd := range params.Forward {
+		path, url, ok := strings.Cut(fwd, "->")
+		if !ok {
+			return fmt.Errorf("invalid forward: %q", fwd)
+		}
+		http.Handle(path, httpcache.SinglePath(&proxy{url: url}, params.CacheTime))
+	}
+
+	return http.ListenAndServe(params.Listen, nil)
+}
+
+type githubReleases struct {
+	url string
+}
+
+func (p *githubReleases) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	log.Println("Fetching", p.url)
+	rels := upgrade.FetchLatestReleases(p.url, "")
+	if rels == nil {
+		http.Error(w, "no releases", http.StatusInternalServerError)
+		return
+	}
 
 	sort.Sort(upgrade.SortByRelease(rels))
 	rels = filterForLatest(rels)
 
-	if err := json.NewEncoder(os.Stdout).Encode(rels); err != nil {
+	buf := new(bytes.Buffer)
-		os.Exit(1)
+	_ = json.NewEncoder(buf).Encode(rels)
+
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Access-Control-Allow-Methods", "GET")
+	w.Write(buf.Bytes())
+}
+
+type proxy struct {
+	url string
+}
+
+func (p *proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	log.Println("Fetching", p.url)
+	req, err := http.NewRequestWithContext(req.Context(), http.MethodGet, p.url, nil)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	defer resp.Body.Close()
+
+	ct := resp.Header.Get("Content-Type")
+	w.Header().Set("Content-Type", ct)
+	if resp.StatusCode == http.StatusOK {
+		w.Header().Set("Cache-Control", "public, max-age=900")
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+		w.Header().Set("Access-Control-Allow-Methods", "GET")
+	}
+	w.WriteHeader(resp.StatusCode)
+	if strings.HasPrefix(ct, "application/json") {
+		// Special JSON handling; clean it up a bit.
+		var v interface{}
+		if err := json.NewDecoder(resp.Body).Decode(&v); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		_ = json.NewEncoder(w).Encode(v)
+	} else {
+		_, _ = io.Copy(w, resp.Body)
 	}
 }
 

cmd/stvanity/main.go

@@ -37,14 +37,14 @@ type result struct {
 
 func main() {
 	flag.Parse()
-	prefix := strings.ToUpper(strings.Replace(flag.Arg(0), "-", "", -1))
+	prefix := strings.ToUpper(strings.ReplaceAll(flag.Arg(0), "-", ""))
 	if len(prefix) > 7 {
 		prefix = prefix[:7] + "-" + prefix[7:]
 	}
 
 	found := make(chan result)
 	stop := make(chan struct{})
-	var count int64
+	var count atomic.Int64
 
 	// Print periodic progress reports.
 	go printProgress(prefix, &count)
@@ -72,7 +72,7 @@ func main() {
 // Try certificates until one is found that has the prefix at the start of
 // the resulting device ID. Increments count atomically, sends the result to
 // found, returns when stop is closed.
-func generatePrefixed(prefix string, count *int64, found chan<- result, stop <-chan struct{}) {
+func generatePrefixed(prefix string, count *atomic.Int64, found chan<- result, stop <-chan struct{}) {
 	notBefore := time.Now()
 	notAfter := time.Date(2049, 12, 31, 23, 59, 59, 0, time.UTC)
 
@@ -109,7 +109,7 @@ func generatePrefixed(prefix string, count *int64, found chan<- result, stop <-c
 		}
 
 		id := protocol.NewDeviceID(derBytes)
-		atomic.AddInt64(count, 1)
+		count.Add(1)
 
 		if strings.HasPrefix(id.String(), prefix) {
 			select {
@@ -121,7 +121,7 @@ func generatePrefixed(prefix string, count *int64, found chan<- result, stop <-c
 	}
 }
 
-func printProgress(prefix string, count *int64) {
+func printProgress(prefix string, count *atomic.Int64) {
 	started := time.Now()
 	wantBits := 5 * len(prefix)
 	if wantBits > 63 {
@@ -132,7 +132,7 @@ func printProgress(prefix string, count *int64) {
 	fmt.Printf("Want %d bits for prefix %q, about %.2g certs to test (statistically speaking)\n", wantBits, prefix, expectedIterations)
 
 	for range time.NewTicker(15 * time.Second).C {
-		tried := atomic.LoadInt64(count)
+		tried := count.Load()
 		elapsed := time.Since(started)
 		rate := float64(tried) / elapsed.Seconds()
 		expected := timeStr(expectedIterations / rate)

cmd/stwatchfile/main.go

@@ -7,31 +7,15 @@
 package main
 
 import (
-	"bytes"
-	"crypto/md5"
 	"flag"
 	"fmt"
 	"io"
 	"os"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
-func getmd5(filePath string) ([]byte, error) {
-	var result []byte
-	file, err := os.Open(filePath)
-	if err != nil {
-		return result, err
-	}
-	defer file.Close()
-
-	hash := md5.New()
-	if _, err := io.Copy(hash, file); err != nil {
-		return result, err
-	}
-
-	return hash.Sum(result), nil
-}
-
 func main() {
 	period := flag.Duration("period", 200*time.Millisecond, "Sleep period between checks")
 	flag.Parse()
@@ -46,7 +30,7 @@ func main() {
 	exists := true
 	size := int64(0)
 	mtime := time.Time{}
-	hash := []byte{}
+	var hash [sha256.Size]byte
 
 	for {
 		time.Sleep(*period)
@@ -72,7 +56,7 @@ func main() {
 		if !exists {
 			size = 0
 			mtime = time.Time{}
-			hash = []byte{}
+			hash = [sha256.Size]byte{}
 			continue
 		}
 
@@ -83,12 +67,12 @@ func main() {
 		newSize := fi.Size()
 		newMtime := fi.ModTime()
 
-		newHash, err := getmd5(file)
+		newHash, err := sha256file(file)
 		if err != nil {
-			fmt.Println("getmd5:", err)
+			fmt.Println("sha256file:", err)
 		}
 
-		if newSize != size || newMtime != mtime || !bytes.Equal(newHash, hash) {
+		if newSize != size || newMtime != mtime || newHash != hash {
 			fmt.Println(file, "Size:", newSize, "Mtime:", newMtime, "Hash:", fmt.Sprintf("%x", newHash))
 			hash = newHash
 			size = newSize
@@ -96,3 +80,18 @@ func main() {
 		}
 	}
 }
+
+func sha256file(fname string) (hash [sha256.Size]byte, err error) {
+	f, err := os.Open(fname)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	io.Copy(h, f)
+	hb := h.Sum(nil)
+	copy(hash[:], hb)
+
+	return
+}

cmd/syncthing/blockprof.go

@@ -15,19 +15,17 @@ import (
 	"time"
 )
 
-func init() {
+func startBlockProfiler() {
-	if innerProcess && os.Getenv("STBLOCKPROFILE") != "" {
+	profiler := pprof.Lookup("block")
-		profiler := pprof.Lookup("block")
+	if profiler == nil {
-		if profiler == nil {
+		panic("Couldn't find block profiler")
-			panic("Couldn't find block profiler")
-		}
-		l.Debugln("Starting block profiling")
-		go func() {
-			err := saveBlockingProfiles(profiler) // Only returns on error
-			l.Warnln("Block profiler failed:", err)
-			panic("Block profiler failed")
-		}()
 	}
+	l.Debugln("Starting block profiling")
+	go func() {
+		err := saveBlockingProfiles(profiler) // Only returns on error
+		l.Warnln("Block profiler failed:", err)
+		panic("Block profiler failed")
+	}()
 }
 
 func saveBlockingProfiles(profiler *pprof.Profile) error {

cmd/syncthing/cli/client.go

@@ -0,0 +1,172 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strings"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/locations"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+type APIClient interface {
+	Get(url string) (*http.Response, error)
+	Post(url, body string) (*http.Response, error)
+	PutJSON(url string, o interface{}) (*http.Response, error)
+}
+
+type apiClient struct {
+	http.Client
+	cfg    config.GUIConfiguration
+	apikey string
+}
+
+type apiClientFactory struct {
+	cfg config.GUIConfiguration
+}
+
+func (f *apiClientFactory) getClient() (APIClient, error) {
+	// Now if the API key and address is not provided (we are not connecting to a remote instance),
+	// try to rip it out of the config.
+	if f.cfg.RawAddress == "" && f.cfg.APIKey == "" {
+		var err error
+		f.cfg, err = loadGUIConfig()
+		if err != nil {
+			return nil, err
+		}
+	} else if f.cfg.Address() == "" || f.cfg.APIKey == "" {
+		return nil, errors.New("Both --gui-address and --gui-apikey should be specified")
+	}
+
+	httpClient := http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial(f.cfg.Network(), f.cfg.Address())
+			},
+		},
+	}
+	return &apiClient{
+		Client: httpClient,
+		cfg:    f.cfg,
+		apikey: f.cfg.APIKey,
+	}, nil
+}
+
+func loadGUIConfig() (config.GUIConfiguration, error) {
+	// Load the certs and get the ID
+	cert, err := tls.LoadX509KeyPair(
+		locations.Get(locations.CertFile),
+		locations.Get(locations.KeyFile),
+	)
+	if err != nil {
+		return config.GUIConfiguration{}, fmt.Errorf("reading device ID: %w", err)
+	}
+
+	myID := protocol.NewDeviceID(cert.Certificate[0])
+
+	// Load the config
+	cfg, _, err := config.Load(locations.Get(locations.ConfigFile), myID, events.NoopLogger)
+	if err != nil {
+		return config.GUIConfiguration{}, fmt.Errorf("loading config: %w", err)
+	}
+
+	guiCfg := cfg.GUI()
+
+	if guiCfg.Address() == "" {
+		return config.GUIConfiguration{}, errors.New("Could not find GUI Address")
+	}
+
+	if guiCfg.APIKey == "" {
+		return config.GUIConfiguration{}, errors.New("Could not find GUI API key")
+	}
+
+	return guiCfg, nil
+}
+
+func (c *apiClient) Endpoint() string {
+	if c.cfg.Network() == "unix" {
+		return "http://unix/"
+	}
+	url := c.cfg.URL()
+	if !strings.HasSuffix(url, "/") {
+		url += "/"
+	}
+	return url
+}
+
+func (c *apiClient) Do(req *http.Request) (*http.Response, error) {
+	req.Header.Set("X-API-Key", c.apikey)
+	resp, err := c.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	return resp, checkResponse(resp)
+}
+
+func (c *apiClient) Request(url, method string, r io.Reader) (*http.Response, error) {
+	request, err := http.NewRequest(method, c.Endpoint()+"rest/"+url, r)
+	if err != nil {
+		return nil, err
+	}
+	return c.Do(request)
+}
+
+func (c *apiClient) RequestString(url, method, data string) (*http.Response, error) {
+	return c.Request(url, method, bytes.NewBufferString(data))
+}
+
+func (c *apiClient) RequestJSON(url, method string, o interface{}) (*http.Response, error) {
+	data, err := json.Marshal(o)
+	if err != nil {
+		return nil, err
+	}
+	return c.Request(url, method, bytes.NewBuffer(data))
+}
+
+func (c *apiClient) Get(url string) (*http.Response, error) {
+	return c.RequestString(url, "GET", "")
+}
+
+func (c *apiClient) Post(url, body string) (*http.Response, error) {
+	return c.RequestString(url, "POST", body)
+}
+
+func (c *apiClient) PutJSON(url string, o interface{}) (*http.Response, error) {
+	return c.RequestJSON(url, "PUT", o)
+}
+
+var errNotFound = errors.New("invalid endpoint or API call")
+
+func checkResponse(response *http.Response) error {
+	if response.StatusCode == http.StatusNotFound {
+		return errNotFound
+	} else if response.StatusCode == http.StatusUnauthorized {
+		return errors.New("invalid API key")
+	} else if response.StatusCode != http.StatusOK {
+		data, err := responseToBArray(response)
+		if err != nil {
+			return err
+		}
+		body := strings.TrimSpace(string(data))
+		return fmt.Errorf("unexpected HTTP status returned: %s\n%s", response.Status, body)
+	}
+	return nil
+}

cmd/syncthing/cli/config.go

@@ -0,0 +1,87 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"reflect"
+
+	"github.com/AudriusButkevicius/recli"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/urfave/cli"
+)
+
+type configHandler struct {
+	original, cfg config.Configuration
+	client        APIClient
+	err           error
+}
+
+func getConfigCommand(f *apiClientFactory) (cli.Command, error) {
+	h := new(configHandler)
+	h.client, h.err = f.getClient()
+	if h.err == nil {
+		h.cfg, h.err = getConfig(h.client)
+	}
+	h.original = h.cfg.Copy()
+
+	// Copy the config and set the default flags
+	recliCfg := recli.DefaultConfig
+	recliCfg.IDTag.Name = "xml"
+	recliCfg.SkipTag.Name = "json"
+
+	commands, err := recli.New(recliCfg).Construct(&h.cfg)
+	if err != nil {
+		return cli.Command{}, fmt.Errorf("config reflect: %w", err)
+	}
+
+	return cli.Command{
+		Name:        "config",
+		HideHelp:    true,
+		Usage:       "Configuration modification command group",
+		Subcommands: commands,
+		Before:      h.configBefore,
+		After:       h.configAfter,
+	}, nil
+}
+
+func (h *configHandler) configBefore(c *cli.Context) error {
+	for _, arg := range c.Args() {
+		if arg == "--help" || arg == "-h" {
+			return nil
+		}
+	}
+	return h.err
+}
+
+func (h *configHandler) configAfter(_ *cli.Context) error {
+	if h.err != nil {
+		// Error was already returned in configBefore
+		return nil
+	}
+	if reflect.DeepEqual(h.cfg, h.original) {
+		return nil
+	}
+	body, err := json.MarshalIndent(h.cfg, "", "  ")
+	if err != nil {
+		return err
+	}
+	resp, err := h.client.Post("system/config", string(body))
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode != 200 {
+		body, err := responseToBArray(resp)
+		if err != nil {
+			return err
+		}
+		return errors.New(string(body))
+	}
+	return nil
+}

cmd/syncthing/cli/debug.go

@@ -0,0 +1,55 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/urfave/cli"
+)
+
+var debugCommand = cli.Command{
+	Name:     "debug",
+	HideHelp: true,
+	Usage:    "Debug command group",
+	Subcommands: []cli.Command{
+		{
+			Name:      "file",
+			Usage:     "Show information about a file (or directory/symlink)",
+			ArgsUsage: "FOLDER-ID PATH",
+			Action:    expects(2, debugFile()),
+		},
+		indexCommand,
+		{
+			Name:      "profile",
+			Usage:     "Save a profile to help figuring out what Syncthing does.",
+			ArgsUsage: "cpu | heap",
+			Action:    expects(1, profile()),
+		},
+	},
+}
+
+func debugFile() cli.ActionFunc {
+	return func(c *cli.Context) error {
+		query := make(url.Values)
+		query.Set("folder", c.Args()[0])
+		query.Set("file", normalizePath(c.Args()[1]))
+		return indexDumpOutput("debug/file?" + query.Encode())(c)
+	}
+}
+
+func profile() cli.ActionFunc {
+	return func(c *cli.Context) error {
+		switch t := c.Args()[0]; t {
+		case "cpu", "heap":
+			return saveToFile(fmt.Sprintf("debug/%vprof", c.Args()[0]))(c)
+		default:
+			return fmt.Errorf("expected cpu or heap as argument, got %v", t)
+		}
+	}
+}

cmd/syncthing/cli/errors.go

@@ -4,7 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
 	"errors"
@@ -22,12 +22,12 @@ var errorsCommand = cli.Command{
 		{
 			Name:   "show",
 			Usage:  "Show pending errors",
-			Action: expects(0, dumpOutput("system/error")),
+			Action: expects(0, indexDumpOutput("system/error")),
 		},
 		{
 			Name:      "push",
 			Usage:     "Push an error to active clients",
-			ArgsUsage: "[error message]",
+			ArgsUsage: "ERROR-MESSAGE",
 			Action:    expects(1, errorsPush),
 		},
 		{
@@ -39,7 +39,7 @@ var errorsCommand = cli.Command{
 }
 
 func errorsPush(c *cli.Context) error {
-	client := c.App.Metadata["client"].(*APIClient)
+	client := c.App.Metadata["client"].(APIClient)
 	errStr := strings.Join(c.Args(), " ")
 	response, err := client.Post("system/error", strings.TrimSpace(errStr))
 	if err != nil {

cmd/syncthing/cli/index.go

@@ -0,0 +1,38 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"github.com/urfave/cli"
+)
+
+var indexCommand = cli.Command{
+	Name:  "index",
+	Usage: "Show information about the index (database)",
+	Subcommands: []cli.Command{
+		{
+			Name:   "dump",
+			Usage:  "Print the entire db",
+			Action: expects(0, indexDump),
+		},
+		{
+			Name:   "dump-size",
+			Usage:  "Print the db size of different categories of information",
+			Action: expects(0, indexDumpSize),
+		},
+		{
+			Name:   "check",
+			Usage:  "Check the database for inconsistencies",
+			Action: expects(0, indexCheck),
+		},
+		{
+			Name:   "account",
+			Usage:  "Print key and value size statistics per key type",
+			Action: expects(0, indexAccount),
+		},
+	},
+}

cmd/syncthing/cli/index_accounting.go

@@ -4,22 +4,26 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
 	"fmt"
-	"log"
 	"os"
 	"text/tabwriter"
 
-	"github.com/syncthing/syncthing/lib/db/backend"
+	"github.com/urfave/cli"
 )
 
-// account prints key and data size statistics per class
+// indexAccount prints key and data size statistics per class
-func account(ldb backend.Backend) {
+func indexAccount(*cli.Context) error {
+	ldb, err := getDB()
+	if err != nil {
+		return err
+	}
+
 	it, err := ldb.NewPrefixIterator(nil)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 
 	var ksizes [256]int
@@ -55,4 +59,6 @@ func account(ldb backend.Backend) {
 	}
 	fmt.Fprintf(tw, "Total\t%d items,\t%d KB keys +\t%d KB data.\t\n", toti, totks/1000, totds/1000)
 	tw.Flush()
+
+	return nil
 }

cmd/syncthing/cli/index_dump.go

@@ -0,0 +1,160 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"encoding/binary"
+	"fmt"
+	"time"
+
+	"github.com/urfave/cli"
+
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func indexDump(*cli.Context) error {
+	ldb, err := getDB()
+	if err != nil {
+		return err
+	}
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		return err
+	}
+	for it.Next() {
+		key := it.Key()
+		switch key[0] {
+		case db.KeyTypeDevice:
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+			fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
+
+			var f protocol.FileInfo
+			err := f.Unmarshal(it.Value())
+			if err != nil {
+				return err
+			}
+			fmt.Printf(" V:%v\n", f)
+
+		case db.KeyTypeGlobal:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			var flv db.VersionList
+			flv.Unmarshal(it.Value())
+			fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
+
+		case db.KeyTypeBlock:
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[1+4 : 1+4+32]
+			name := nulString(key[1+4+32:])
+			fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
+
+		case db.KeyTypeDeviceStatistic:
+			fmt.Printf("[dstat] K:%x V:%x\n", key, it.Value())
+
+		case db.KeyTypeFolderStatistic:
+			fmt.Printf("[fstat] K:%x V:%x\n", key, it.Value())
+
+		case db.KeyTypeVirtualMtime:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			val := it.Value()
+			var realTime, virtualTime time.Time
+			realTime.UnmarshalBinary(val[:len(val)/2])
+			virtualTime.UnmarshalBinary(val[len(val)/2:])
+			fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, realTime, virtualTime)
+
+		case db.KeyTypeFolderIdx:
+			key := binary.BigEndian.Uint32(key[1:])
+			fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
+
+		case db.KeyTypeDeviceIdx:
+			key := binary.BigEndian.Uint32(key[1:])
+			val := it.Value()
+			device := "<nil>"
+			if len(val) > 0 {
+				dev, err := protocol.DeviceIDFromBytes(val)
+				if err != nil {
+					device = fmt.Sprintf("<invalid %d bytes>", len(val))
+				} else {
+					device = dev.String()
+				}
+			}
+			fmt.Printf("[deviceidx] K:%d V:%s\n", key, device)
+
+		case db.KeyTypeIndexID:
+			device := binary.BigEndian.Uint32(key[1:])
+			folder := binary.BigEndian.Uint32(key[5:])
+			fmt.Printf("[indexid] D:%d F:%d I:%x\n", device, folder, it.Value())
+
+		case db.KeyTypeFolderMeta:
+			folder := binary.BigEndian.Uint32(key[1:])
+			fmt.Printf("[foldermeta] F:%d", folder)
+			var cs db.CountsSet
+			if err := cs.Unmarshal(it.Value()); err != nil {
+				fmt.Printf(" (invalid)\n")
+			} else {
+				fmt.Printf(" V:%v\n", cs)
+			}
+
+		case db.KeyTypeMiscData:
+			fmt.Printf("[miscdata] K:%q V:%q\n", key[1:], it.Value())
+
+		case db.KeyTypeSequence:
+			folder := binary.BigEndian.Uint32(key[1:])
+			seq := binary.BigEndian.Uint64(key[5:])
+			fmt.Printf("[sequence] F:%d S:%d V:%q\n", folder, seq, it.Value())
+
+		case db.KeyTypeNeed:
+			folder := binary.BigEndian.Uint32(key[1:])
+			file := string(key[5:])
+			fmt.Printf("[need] F:%d V:%q\n", folder, file)
+
+		case db.KeyTypeBlockList:
+			fmt.Printf("[blocklist] H:%x\n", key[1:])
+
+		case db.KeyTypeBlockListMap:
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[5:37]
+			fileName := string(key[37:])
+			fmt.Printf("[blocklistmap] F:%d H:%x N:%s\n", folder, hash, fileName)
+
+		case db.KeyTypeVersion:
+			fmt.Printf("[version] H:%x", key[1:])
+			var v protocol.Vector
+			err := v.Unmarshal(it.Value())
+			if err != nil {
+				fmt.Printf(" (invalid)\n")
+			} else {
+				fmt.Printf(" V:%v\n", v)
+			}
+
+		case db.KeyTypePendingFolder:
+			device := binary.BigEndian.Uint32(key[1:])
+			folder := string(key[5:])
+			var of db.ObservedFolder
+			of.Unmarshal(it.Value())
+			fmt.Printf("[pendingFolder] D:%d F:%s V:%v\n", device, folder, of)
+
+		case db.KeyTypePendingDevice:
+			device := "<invalid>"
+			dev, err := protocol.DeviceIDFromBytes(key[1:])
+			if err == nil {
+				device = dev.String()
+			}
+			var od db.ObservedDevice
+			od.Unmarshal(it.Value())
+			fmt.Printf("[pendingDevice] D:%v V:%v\n", device, od)
+
+		default:
+			fmt.Printf("[??? %d]\n  %x\n  %x\n", key[0], key, it.Value())
+		}
+	}
+	return nil
+}

cmd/syncthing/cli/index_dumpsize.go

@@ -4,51 +4,38 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
-	"container/heap"
 	"encoding/binary"
 	"fmt"
-	"log"
+	"sort"
+
+	"github.com/urfave/cli"
 
 	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/db/backend"
 )
 
-type SizedElement struct {
+func indexDumpSize(*cli.Context) error {
-	key  string
+	type sizedElement struct {
-	size int
+		key  string
-}
+		size int
+	}
 
-type ElementHeap []SizedElement
+	ldb, err := getDB()
-
+	if err != nil {
-func (h ElementHeap) Len() int           { return len(h) }
+		return err
-func (h ElementHeap) Less(i, j int) bool { return h[i].size > h[j].size }
+	}
-func (h ElementHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
-
-func (h *ElementHeap) Push(x interface{}) {
-	*h = append(*h, x.(SizedElement))
-}
-
-func (h *ElementHeap) Pop() interface{} {
-	old := *h
-	n := len(old)
-	x := old[n-1]
-	*h = old[0 : n-1]
-	return x
-}
-
-func dumpsize(ldb backend.Backend) {
-	h := &ElementHeap{}
-	heap.Init(h)
 
 	it, err := ldb.NewPrefixIterator(nil)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
-	var ele SizedElement
+
+	var elems []sizedElement
 	for it.Next() {
+		var ele sizedElement
+
 		key := it.Key()
 		switch key[0] {
 		case db.KeyTypeDevice:
@@ -89,11 +76,15 @@ func dumpsize(ldb backend.Backend) {
 			ele.key = fmt.Sprintf("UNKNOWN:%x", key)
 		}
 		ele.size = len(it.Value())
-		heap.Push(h, ele)
+		elems = append(elems, ele)
 	}
 
-	for h.Len() > 0 {
+	sort.Slice(elems, func(i, j int) bool {
-		ele = heap.Pop(h).(SizedElement)
+		return elems[i].size > elems[j].size
+	})
+	for _, ele := range elems {
 		fmt.Println(ele.key, ele.size)
 	}
+
+	return nil
 }

cmd/syncthing/cli/index_idxck.go

@@ -4,17 +4,18 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
 	"bytes"
 	"encoding/binary"
+	"errors"
 	"fmt"
-	"log"
 	"sort"
 
+	"github.com/urfave/cli"
+
 	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/db/backend"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -34,7 +35,12 @@ type sequenceKey struct {
 	sequence uint64
 }
 
-func idxck(ldb backend.Backend) (success bool) {
+func indexCheck(*cli.Context) (err error) {
+	ldb, err := getDB()
+	if err != nil {
+		return err
+	}
+
 	folders := make(map[uint32]string)
 	devices := make(map[uint32]string)
 	deviceToIDs := make(map[string]uint32)
@@ -43,13 +49,24 @@ func idxck(ldb backend.Backend) (success bool) {
 	sequences := make(map[sequenceKey]string)
 	needs := make(map[globalKey]struct{})
 	blocklists := make(map[string]struct{})
+	versions := make(map[string]protocol.Vector)
 	usedBlocklists := make(map[string]struct{})
+	usedVersions := make(map[string]struct{})
 	var localDeviceKey uint32
-	success = true
+	success := true
+	defer func() {
+		if err == nil {
+			if success {
+				fmt.Println("Index check completed successfully.")
+			} else {
+				err = errors.New("Inconsistencies found in the index")
+			}
+		}
+	}()
 
 	it, err := ldb.NewPrefixIterator(nil)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 	for it.Next() {
 		key := it.Key()
@@ -106,6 +123,16 @@ func idxck(ldb backend.Backend) (success bool) {
 		case db.KeyTypeBlockList:
 			hash := string(key[1:])
 			blocklists[hash] = struct{}{}
+
+		case db.KeyTypeVersion:
+			hash := string(key[1:])
+			var v protocol.Vector
+			if err := v.Unmarshal(it.Value()); err != nil {
+				fmt.Println("Unable to unmarshal Vector:", err)
+				success = false
+				continue
+			}
+			versions[hash] = v
 		}
 	}
 
@@ -157,6 +184,23 @@ func idxck(ldb backend.Backend) (success bool) {
 				usedBlocklists[key] = struct{}{}
 			}
 		}
+
+		if fi.VersionHash != nil {
+			key := string(fi.VersionHash)
+			if _, ok := versions[key]; !ok {
+				fmt.Printf("Missing version vector for file %q, version hash %x\n", fi.Name, fi.VersionHash)
+				success = false
+			} else {
+				usedVersions[key] = struct{}{}
+			}
+		}
+
+		_, ok := globals[globalKey{fk.folder, fk.name}]
+		if !ok {
+			fmt.Printf("Missing global for file %q\n", fi.Name)
+			success = false
+			continue
+		}
 	}
 
 	// Aggregate the ranges of missing sequence entries, print them
@@ -190,10 +234,10 @@ func idxck(ldb backend.Backend) (success bool) {
 			fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
 			success = false
 		}
-		for i, fv := range vl.Versions {
+		checkGlobal := func(i int, device []byte, version protocol.Vector, invalid, deleted bool) {
-			dev, ok := deviceToIDs[string(fv.Device)]
+			dev, ok := deviceToIDs[string(device)]
 			if !ok {
-				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, fv.Device)
+				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, device)
 				success = false
 			}
 			fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
@@ -201,14 +245,31 @@ func idxck(ldb backend.Backend) (success bool) {
 				fmt.Printf("VersionList %q, folder %q, entry %d refers to unknown FileInfo\n", gk.name, folder, i)
 				success = false
 			}
-			if !fi.Version.Equal(fv.Version) {
+
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Version, fi.Version)
+			fiv := fi.Version
+			if fi.VersionHash != nil {
+				fiv = versions[string(fi.VersionHash)]
+			}
+			if !fiv.Equal(version) {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, version, fi.Version)
 				success = false
 			}
-			if fi.IsInvalid() != fv.Invalid {
+			if fi.IsInvalid() != invalid {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Invalid, fi.IsInvalid())
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, invalid, fi.IsInvalid())
 				success = false
 			}
+			if fi.IsDeleted() != deleted {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo deleted mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, deleted, fi.IsDeleted())
+				success = false
+			}
+		}
+		for i, fv := range vl.RawVersions {
+			for _, device := range fv.Devices {
+				checkGlobal(i, device, fv.Version, false, fv.Deleted)
+			}
+			for _, device := range fv.InvalidDevices {
+				checkGlobal(i, device, fv.Version, true, fv.Deleted)
+			}
 		}
 
 		// If we need this file we should have a need entry for it. False
@@ -217,7 +278,9 @@ func idxck(ldb backend.Backend) (success bool) {
 		if needsLocally(vl) {
 			_, ok := needs[gk]
 			if !ok {
-				dev := deviceToIDs[string(vl.Versions[0].Device)]
+				fv, _ := vl.GetGlobal()
+				devB, _ := fv.FirstDevice()
+				dev := deviceToIDs[string(devB)]
 				fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
 				if !fi.IsDeleted() && !fi.IsIgnored() {
 					fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
@@ -277,20 +340,18 @@ func idxck(ldb backend.Backend) (success bool) {
 	if d := len(blocklists) - len(usedBlocklists); d > 0 {
 		fmt.Printf("%d block list entries out of %d needs GC\n", d, len(blocklists))
 	}
+	if d := len(versions) - len(usedVersions); d > 0 {
+		fmt.Printf("%d version entries out of %d needs GC\n", d, len(versions))
+	}
 
-	return
+	return nil
 }
 
 func needsLocally(vl db.VersionList) bool {
-	var lv *protocol.Vector
+	gfv, gok := vl.GetGlobal()
-	for _, fv := range vl.Versions {
+	if !gok { // That's weird, but we hardly need something non-existent
-		if bytes.Equal(fv.Device, protocol.LocalDeviceID[:]) {
+		return false
-			lv = &fv.Version
-			break
-		}
 	}
-	if lv == nil {
+	fv, ok := vl.Get(protocol.LocalDeviceID[:])
-		return true // proviosinally, it looks like we need the file
+	return db.Need(gfv, ok, fv.Version)
-	}
-	return !lv.GreaterEqual(vl.Versions[0].Version)
 }

cmd/syncthing/cli/main.go

@@ -0,0 +1,172 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/alecthomas/kong"
+	"github.com/flynn-archive/go-shlex"
+	"github.com/urfave/cli"
+
+	"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
+	"github.com/syncthing/syncthing/lib/config"
+)
+
+type preCli struct {
+	GUIAddress string `name:"gui-address"`
+	GUIAPIKey  string `name:"gui-apikey"`
+	HomeDir    string `name:"home"`
+	ConfDir    string `name:"config"`
+	DataDir    string `name:"data"`
+}
+
+func Run() error {
+	// This is somewhat a hack around a chicken and egg problem. We need to set
+	// the home directory and potentially other flags to know where the
+	// syncthing instance is running in order to get it's config ... which we
+	// then use to construct the actual CLI ... at which point it's too late to
+	// add flags there...
+	c := preCli{}
+	parseFlags(&c)
+	return runInternal(c, os.Args)
+}
+
+func RunWithArgs(cliArgs []string) error {
+	c := preCli{}
+	parseFlagsWithArgs(cliArgs, &c)
+	return runInternal(c, cliArgs)
+}
+
+func runInternal(c preCli, cliArgs []string) error {
+	// Not set as default above because the strings can be really long.
+	err := cmdutil.SetConfigDataLocationsFromFlags(c.HomeDir, c.ConfDir, c.DataDir)
+	if err != nil {
+		return fmt.Errorf("Command line options: %w", err)
+	}
+	clientFactory := &apiClientFactory{
+		cfg: config.GUIConfiguration{
+			RawAddress: c.GUIAddress,
+			APIKey:     c.GUIAPIKey,
+		},
+	}
+
+	configCommand, err := getConfigCommand(clientFactory)
+	if err != nil {
+		return err
+	}
+
+	// Implement the same flags at the upper CLI, but do nothing with them.
+	// This is so that the usage text is the same
+	fakeFlags := []cli.Flag{
+		cli.StringFlag{
+			Name:  "gui-address",
+			Usage: "Override GUI address to `URL` (e.g. \"192.0.2.42:8443\")",
+		},
+		cli.StringFlag{
+			Name:  "gui-apikey",
+			Usage: "Override GUI API key to `API-KEY`",
+		},
+		cli.StringFlag{
+			Name:  "home",
+			Usage: "Set configuration and data directory to `PATH`",
+		},
+		cli.StringFlag{
+			Name:  "config",
+			Usage: "Set configuration directory (config and keys) to `PATH`",
+		},
+		cli.StringFlag{
+			Name:  "data",
+			Usage: "Set data directory (database and logs) to `PATH`",
+		},
+	}
+
+	// Construct the actual CLI
+	app := cli.NewApp()
+	app.Author = "The Syncthing Authors"
+	app.Metadata = map[string]interface{}{
+		"clientFactory": clientFactory,
+	}
+	app.Commands = []cli.Command{{
+		Name:  "cli",
+		Usage: "Syncthing command line interface",
+		Flags: fakeFlags,
+		Subcommands: []cli.Command{
+			configCommand,
+			showCommand,
+			operationCommand,
+			errorsCommand,
+			debugCommand,
+			{
+				Name:     "-",
+				HideHelp: true,
+				Usage:    "Read commands from stdin",
+				Action: func(ctx *cli.Context) error {
+					if ctx.NArg() > 0 {
+						return errors.New("command does not expect any arguments")
+					}
+
+					// Drop the `-` not to recurse into self.
+					args := make([]string, len(cliArgs)-1)
+					copy(args, cliArgs)
+
+					fmt.Println("Reading commands from stdin...", args)
+					scanner := bufio.NewScanner(os.Stdin)
+					for scanner.Scan() {
+						input, err := shlex.Split(scanner.Text())
+						if err != nil {
+							return fmt.Errorf("parsing input: %w", err)
+						}
+						if len(input) == 0 {
+							continue
+						}
+						err = app.Run(append(args, input...))
+						if err != nil {
+							return err
+						}
+					}
+					return scanner.Err()
+				},
+			},
+		},
+	}}
+
+	return app.Run(cliArgs)
+}
+
+func parseFlags(c *preCli) error {
+	// kong only needs to parse the global arguments after "cli" and before the
+	// subcommand (if any).
+	if len(os.Args) <= 2 {
+		return nil
+	}
+	return parseFlagsWithArgs(os.Args[2:], c)
+}
+
+func parseFlagsWithArgs(args []string, c *preCli) error {
+	for i := 0; i < len(args); i++ {
+		if !strings.HasPrefix(args[i], "--") {
+			args = args[:i]
+			break
+		}
+		if !strings.Contains(args[i], "=") {
+			i++
+		}
+	}
+	// We don't want kong to print anything nor os.Exit (e.g. on -h)
+	parser, err := kong.New(c, kong.Writers(io.Discard, io.Discard), kong.Exit(func(int) {}))
+	if err != nil {
+		return err
+	}
+	_, err = parser.Parse(args)
+	return err
+}

cmd/syncthing/cli/operations.go

@@ -4,11 +4,16 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
+	"bufio"
+	"errors"
 	"fmt"
+	"path/filepath"
 
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/urfave/cli"
 )
 
@@ -27,11 +32,6 @@ var operationCommand = cli.Command{
 			Usage:  "Shutdown syncthing",
 			Action: expects(0, emptyPost("system/shutdown")),
 		},
-		{
-			Name:   "reset",
-			Usage:  "Reset syncthing deleting all folders and devices",
-			Action: expects(0, emptyPost("system/reset")),
-		},
 		{
 			Name:   "upgrade",
 			Usage:  "Upgrade syncthing (if a newer version is available)",
@@ -39,15 +39,24 @@ var operationCommand = cli.Command{
 		},
 		{
 			Name:      "folder-override",
-			Usage:     "Override changes on folder (remote for sendonly, local for receiveonly)",
+			Usage:     "Override changes on folder (remote for sendonly, local for receiveonly). WARNING: Destructive - deletes/changes your data.",
-			ArgsUsage: "[folder id]",
+			ArgsUsage: "FOLDER-ID",
 			Action:    expects(1, foldersOverride),
 		},
+		{
+			Name:      "default-ignores",
+			Usage:     "Set the default ignores (config) from a file",
+			ArgsUsage: "PATH",
+			Action:    expects(1, setDefaultIgnores),
+		},
 	},
 }
 
 func foldersOverride(c *cli.Context) error {
-	client := c.App.Metadata["client"].(*APIClient)
+	client, err := getClientFactory(c).getClient()
+	if err != nil {
+		return err
+	}
 	cfg, err := getConfig(client)
 	if err != nil {
 		return err
@@ -69,10 +78,36 @@ func foldersOverride(c *cli.Context) error {
 				if body != "" {
 					errStr += "\nBody: " + body
 				}
-				return fmt.Errorf(errStr)
+				return errors.New(errStr)
 			}
 			return nil
 		}
 	}
-	return fmt.Errorf("Folder " + rid + " not found")
+	return fmt.Errorf("Folder %q not found", rid)
+}
+
+func setDefaultIgnores(c *cli.Context) error {
+	client, err := getClientFactory(c).getClient()
+	if err != nil {
+		return err
+	}
+	dir, file := filepath.Split(c.Args()[0])
+	filesystem := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
+
+	fd, err := filesystem.Open(file)
+	if err != nil {
+		return err
+	}
+	scanner := bufio.NewScanner(fd)
+	var lines []string
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+	}
+	fd.Close()
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+
+	_, err = client.PutJSON("config/defaults/ignores", config.Ignores{Lines: lines})
+	return err
 }

cmd/syncthing/cli/pending.go

@@ -0,0 +1,45 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"net/url"
+
+	"github.com/urfave/cli"
+)
+
+var pendingCommand = cli.Command{
+	Name:     "pending",
+	HideHelp: true,
+	Usage:    "Pending subcommand group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "devices",
+			Usage:  "Show pending devices",
+			Action: expects(0, indexDumpOutput("cluster/pending/devices")),
+		},
+		{
+			Name:  "folders",
+			Usage: "Show pending folders",
+			Flags: []cli.Flag{
+				cli.StringFlag{Name: "device", Usage: "Show pending folders offered by given device"},
+			},
+			Action: expects(0, folders()),
+		},
+	},
+}
+
+func folders() cli.ActionFunc {
+	return func(c *cli.Context) error {
+		if c.String("device") != "" {
+			query := make(url.Values)
+			query.Set("device", c.String("device"))
+			return indexDumpOutput("cluster/pending/folders?" + query.Encode())(c)
+		}
+		return indexDumpOutput("cluster/pending/folders")(c)
+	}
+}

cmd/syncthing/cli/show.go

@@ -4,7 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cli
 
 import (
 	"github.com/urfave/cli"
@@ -18,27 +18,33 @@ var showCommand = cli.Command{
 		{
 			Name:   "version",
 			Usage:  "Show syncthing client version",
-			Action: expects(0, dumpOutput("system/version")),
+			Action: expects(0, indexDumpOutput("system/version")),
 		},
 		{
 			Name:   "config-status",
 			Usage:  "Show configuration status, whether or not a restart is required for changes to take effect",
-			Action: expects(0, dumpOutput("system/config/insync")),
+			Action: expects(0, indexDumpOutput("config/restart-required")),
 		},
 		{
 			Name:   "system",
 			Usage:  "Show system status",
-			Action: expects(0, dumpOutput("system/status")),
+			Action: expects(0, indexDumpOutput("system/status")),
 		},
 		{
 			Name:   "connections",
 			Usage:  "Report about connections to other devices",
-			Action: expects(0, dumpOutput("system/connections")),
+			Action: expects(0, indexDumpOutput("system/connections")),
 		},
+		{
+			Name:   "discovery",
+			Usage:  "Show the discovered addresses of remote devices (from cache of the running syncthing instance)",
+			Action: expects(0, indexDumpOutput("system/discovery")),
+		},
+		pendingCommand,
 		{
 			Name:   "usage",
 			Usage:  "Show usage report",
-			Action: expects(0, dumpOutput("svc/report")),
+			Action: expects(0, indexDumpOutput("svc/report")),
 		},
 	},
 }

cmd/syncthing/cli/utils.go

@@ -0,0 +1,165 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/db/backend"
+	"github.com/syncthing/syncthing/lib/locations"
+	"github.com/urfave/cli"
+)
+
+func responseToBArray(response *http.Response) ([]byte, error) {
+	bytes, err := io.ReadAll(response.Body)
+	if err != nil {
+		return nil, err
+	}
+	return bytes, response.Body.Close()
+}
+
+func emptyPost(url string) cli.ActionFunc {
+	return func(c *cli.Context) error {
+		client, err := getClientFactory(c).getClient()
+		if err != nil {
+			return err
+		}
+		_, err = client.Post(url, "")
+		return err
+	}
+}
+
+func indexDumpOutput(url string) cli.ActionFunc {
+	return func(c *cli.Context) error {
+		client, err := getClientFactory(c).getClient()
+		if err != nil {
+			return err
+		}
+		response, err := client.Get(url)
+		if errors.Is(err, errNotFound) {
+			return errors.New("not found (folder/file not in database)")
+		}
+		if err != nil {
+			return err
+		}
+		return prettyPrintResponse(response)
+	}
+}
+
+func saveToFile(url string) cli.ActionFunc {
+	return func(c *cli.Context) error {
+		client, err := getClientFactory(c).getClient()
+		if err != nil {
+			return err
+		}
+		response, err := client.Get(url)
+		if err != nil {
+			return err
+		}
+		_, params, err := mime.ParseMediaType(response.Header.Get("Content-Disposition"))
+		if err != nil {
+			return err
+		}
+		filename := params["filename"]
+		if filename == "" {
+			return errors.New("Missing filename in response")
+		}
+		bs, err := responseToBArray(response)
+		if err != nil {
+			return err
+		}
+		f, err := os.Create(filename)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		_, err = f.Write(bs)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Wrote results to", filename)
+		return err
+	}
+}
+
+func getConfig(c APIClient) (config.Configuration, error) {
+	cfg := config.Configuration{}
+	response, err := c.Get("system/config")
+	if err != nil {
+		return cfg, err
+	}
+	bytes, err := responseToBArray(response)
+	if err != nil {
+		return cfg, err
+	}
+	err = json.Unmarshal(bytes, &cfg)
+	if err != nil {
+		return config.Configuration{}, err
+	}
+	return cfg, nil
+}
+
+func expects(n int, actionFunc cli.ActionFunc) cli.ActionFunc {
+	return func(ctx *cli.Context) error {
+		if ctx.NArg() != n {
+			plural := ""
+			if n != 1 {
+				plural = "s"
+			}
+			return fmt.Errorf("expected %d argument%s, got %d", n, plural, ctx.NArg())
+		}
+		return actionFunc(ctx)
+	}
+}
+
+func prettyPrintJSON(data interface{}) error {
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	return enc.Encode(data)
+}
+
+func prettyPrintResponse(response *http.Response) error {
+	bytes, err := responseToBArray(response)
+	if err != nil {
+		return err
+	}
+	var data interface{}
+	if err := json.Unmarshal(bytes, &data); err != nil {
+		return err
+	}
+	// TODO: Check flag for pretty print format
+	return prettyPrintJSON(data)
+}
+
+func getDB() (backend.Backend, error) {
+	return backend.OpenLevelDBRO(locations.Get(locations.Database))
+}
+
+func nulString(bs []byte) string {
+	for i := range bs {
+		if bs[i] == 0 {
+			return string(bs[:i])
+		}
+	}
+	return string(bs)
+}
+
+func normalizePath(path string) string {
+	return filepath.ToSlash(filepath.Clean(path))
+}
+
+func getClientFactory(c *cli.Context) *apiClientFactory {
+	return c.App.Metadata["clientFactory"].(*apiClientFactory)
+}

cmd/syncthing/cmdutil/options_common.go

@@ -0,0 +1,16 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cmdutil
+
+// CommonOptions are reused among several subcommands
+type CommonOptions struct {
+	buildCommonOptions
+	ConfDir         string `name:"config" placeholder:"PATH" help:"Set configuration directory (config and keys)"`
+	HomeDir         string `name:"home" placeholder:"PATH" help:"Set configuration and data directory"`
+	NoDefaultFolder bool   `env:"STNODEFAULTFOLDER" help:"Don't create the \"default\" folder on first startup"`
+	SkipPortProbing bool   `help:"Don't try to find free ports for GUI and listen addresses on first startup"`
+}

cmd/syncthing/cmdutil/options_others.go

@@ -1,10 +1,14 @@
-// Copyright (C) 2016 The Syncthing Authors.
+// Copyright (C) 2021 The Syncthing Authors.
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// The existence of this file means we get 0% test coverage rather than no
+//go:build !windows
-// test coverage at all. Remove when implementing an actual test.
+// +build !windows
 
-package stats
+package cmdutil
+
+type buildCommonOptions struct {
+	HideConsole bool `hidden:""`
+}

cmd/syncthing/cmdutil/options_windows.go

@@ -0,0 +1,11 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cmdutil
+
+type buildCommonOptions struct {
+	HideConsole bool `name:"no-console" help:"Hide console window"`
+}

cmd/syncthing/cmdutil/util.go

@@ -0,0 +1,35 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cmdutil
+
+import (
+	"errors"
+
+	"github.com/syncthing/syncthing/lib/locations"
+)
+
+func SetConfigDataLocationsFromFlags(homeDir, confDir, dataDir string) error {
+	homeSet := homeDir != ""
+	confSet := confDir != ""
+	dataSet := dataDir != ""
+	switch {
+	case dataSet != confSet:
+		return errors.New("either both or none of --config and --data must be given, use --home to set both at once")
+	case homeSet && dataSet:
+		return errors.New("--home must not be used together with --config and --data")
+	case homeSet:
+		confDir = homeDir
+		dataDir = homeDir
+		fallthrough
+	case dataSet:
+		if err := locations.SetBaseDir(locations.ConfigBaseDir, confDir); err != nil {
+			return err
+		}
+		return locations.SetBaseDir(locations.DataBaseDir, dataDir)
+	}
+	return nil
+}

cmd/syncthing/crash_reporting.go

@@ -10,7 +10,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -62,7 +61,7 @@ func uploadPanicLogs(ctx context.Context, urlBase, dir string) {
 // the log contents. A HEAD request is made to see if the log has already
 // been reported. If not, a PUT is made with the log contents.
 func uploadPanicLog(ctx context.Context, urlBase, file string) error {
-	data, err := ioutil.ReadFile(file)
+	data, err := os.ReadFile(file)
 	if err != nil {
 		return err
 	}

cmd/syncthing/decrypt/decrypt.go

@@ -0,0 +1,289 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Package decrypt implements the `syncthing decrypt` subcommand.
+package decrypt
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/fs"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/scanner"
+)
+
+type CLI struct {
+	Path       string `arg:"" required:"1" help:"Path to encrypted folder"`
+	To         string `xor:"mode" placeholder:"PATH" help:"Destination directory, when decrypting"`
+	VerifyOnly bool   `xor:"mode" help:"Don't write decrypted files to disk (but verify plaintext hashes)"`
+	Password   string `help:"Folder password for decryption / verification" env:"FOLDER_PASSWORD"`
+	FolderID   string `help:"Folder ID of the encrypted folder, if it cannot be determined automatically"`
+	Continue   bool   `help:"Continue processing next file in case of error, instead of aborting"`
+	Verbose    bool   `help:"Show verbose progress information"`
+	TokenPath  string `placeholder:"PATH" help:"Path to the token file within the folder (used to determine folder ID)"`
+
+	folderKey *[32]byte
+	keyGen    *protocol.KeyGenerator
+}
+
+type storedEncryptionToken struct {
+	FolderID string
+	Token    []byte
+}
+
+func (c *CLI) Run() error {
+	log.SetFlags(0)
+
+	if c.To == "" && !c.VerifyOnly {
+		return errors.New("must set --to or --verify-only")
+	}
+
+	if c.TokenPath == "" {
+		// This is a bit long to show as default in --help
+		c.TokenPath = filepath.Join(config.DefaultMarkerName, config.EncryptionTokenName)
+	}
+
+	if c.FolderID == "" {
+		// We should try to figure out the folder ID
+		folderID, err := c.getFolderID()
+		if err != nil {
+			log.Println("No --folder-id given and couldn't read folder token")
+			return fmt.Errorf("getting folder ID: %w", err)
+		}
+
+		c.FolderID = folderID
+		if c.Verbose {
+			log.Println("Found folder ID:", c.FolderID)
+		}
+	}
+
+	c.keyGen = protocol.NewKeyGenerator()
+	c.folderKey = c.keyGen.KeyFromPassword(c.FolderID, c.Password)
+
+	return c.walk()
+}
+
+// walk finds and processes every file in the encrypted folder
+func (c *CLI) walk() error {
+	srcFs := fs.NewFilesystem(fs.FilesystemTypeBasic, c.Path)
+	var dstFs fs.Filesystem
+	if c.To != "" {
+		dstFs = fs.NewFilesystem(fs.FilesystemTypeBasic, c.To)
+	}
+
+	return srcFs.Walk(".", func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsRegular() {
+			return nil
+		}
+		if fs.IsInternal(path) {
+			return nil
+		}
+
+		return c.withContinue(c.process(srcFs, dstFs, path))
+	})
+}
+
+// If --continue was set we just mention the error and return nil to
+// continue processing.
+func (c *CLI) withContinue(err error) error {
+	if err == nil {
+		return nil
+	}
+	if c.Continue {
+		log.Println("Warning:", err)
+		return nil
+	}
+	return err
+}
+
+// getFolderID returns the folder ID found in the encrypted token, or an
+// error.
+func (c *CLI) getFolderID() (string, error) {
+	tokenPath := filepath.Join(c.Path, c.TokenPath)
+	bs, err := os.ReadFile(tokenPath)
+	if err != nil {
+		return "", fmt.Errorf("reading folder token: %w", err)
+	}
+
+	var tok storedEncryptionToken
+	if err := json.Unmarshal(bs, &tok); err != nil {
+		return "", fmt.Errorf("parsing folder token: %w", err)
+	}
+
+	return tok.FolderID, nil
+}
+
+// process handles the file named path in srcFs, decrypting it into dstFs
+// unless dstFs is nil.
+func (c *CLI) process(srcFs fs.Filesystem, dstFs fs.Filesystem, path string) error {
+	// Which filemode bits to preserve
+	const retainBits = fs.ModePerm | fs.ModeSetgid | fs.ModeSetuid | fs.ModeSticky
+
+	if c.Verbose {
+		log.Printf("Processing %q", path)
+	}
+
+	encFd, err := srcFs.Open(path)
+	if err != nil {
+		return err
+	}
+	defer encFd.Close()
+
+	encFi, err := loadEncryptedFileInfo(encFd)
+	if err != nil {
+		return fmt.Errorf("%s: loading metadata trailer: %w", path, err)
+	}
+
+	// Workaround for a bug in <= v1.15.0-rc.5 where we stored names
+	// in native format, while protocol expects wire format (slashes).
+	encFi.Name = osutil.NormalizedFilename(encFi.Name)
+
+	plainFi, err := protocol.DecryptFileInfo(c.keyGen, *encFi, c.folderKey)
+	if err != nil {
+		return fmt.Errorf("%s: decrypting metadata: %w", path, err)
+	}
+
+	if c.Verbose {
+		log.Printf("Plaintext filename is %q", plainFi.Name)
+	}
+
+	var plainFd fs.File
+	if dstFs != nil {
+		if err := dstFs.MkdirAll(filepath.Dir(plainFi.Name), 0o700); err != nil {
+			return fmt.Errorf("%s: %w", plainFi.Name, err)
+		}
+
+		plainFd, err = dstFs.Create(plainFi.Name)
+		if err != nil {
+			return fmt.Errorf("%s: %w", plainFi.Name, err)
+		}
+		defer plainFd.Close() // also closed explicitly in the return
+		if err := dstFs.Chmod(plainFi.Name, fs.FileMode(plainFi.Permissions&uint32(retainBits))); err != nil {
+			return fmt.Errorf("%s: %w", plainFi.Name, err)
+		}
+	}
+
+	if err := c.decryptFile(encFi, &plainFi, encFd, plainFd); err != nil {
+		// Decrypting the file failed, leaving it in an inconsistent state.
+		// Delete it. Even --continue currently doesn't mean "leave broken
+		// stuff in place", it just means "try the next file instead of
+		// aborting".
+		if plainFd != nil {
+			_ = dstFs.Remove(plainFd.Name())
+		}
+		return fmt.Errorf("%s: %s: %w", path, plainFi.Name, err)
+	} else if c.Verbose {
+		log.Printf("Data verified for %q", plainFi.Name)
+	}
+
+	if plainFd != nil {
+		if err := plainFd.Close(); err != nil {
+			return fmt.Errorf("%s: %w", plainFi.Name, err)
+		}
+		if err := dstFs.Chtimes(plainFi.Name, plainFi.ModTime(), plainFi.ModTime()); err != nil {
+			return fmt.Errorf("%s: %w", plainFi.Name, err)
+		}
+	}
+	return nil
+}
+
+// decryptFile reads, decrypts and verifies all the blocks in src, writing
+// it to dst if dst is non-nil. (If dst is nil it just becomes a
+// read-and-verify operation.)
+func (c *CLI) decryptFile(encFi *protocol.FileInfo, plainFi *protocol.FileInfo, src io.ReaderAt, dst io.WriterAt) error {
+	// The encrypted and plaintext files must consist of an equal number of blocks
+	if len(encFi.Blocks) != len(plainFi.Blocks) {
+		return fmt.Errorf("block count mismatch: encrypted %d != plaintext %d", len(encFi.Blocks), len(plainFi.Blocks))
+	}
+
+	fileKey := c.keyGen.FileKey(plainFi.Name, c.folderKey)
+	for i, encBlock := range encFi.Blocks {
+		// Read the encrypted block
+		buf := make([]byte, encBlock.Size)
+		if _, err := src.ReadAt(buf, encBlock.Offset); err != nil {
+			return fmt.Errorf("encrypted block %d (%d bytes): %w", i, encBlock.Size, err)
+		}
+
+		// Decrypt it
+		dec, err := protocol.DecryptBytes(buf, fileKey)
+		if err != nil {
+			return fmt.Errorf("encrypted block %d (%d bytes): %w", i, encBlock.Size, err)
+		}
+
+		// Verify the block size against the expected plaintext
+		plainBlock := plainFi.Blocks[i]
+		if i == len(plainFi.Blocks)-1 && len(dec) > plainBlock.Size {
+			// The last block might be padded, which is fine (we skip the padding)
+			dec = dec[:plainBlock.Size]
+		} else if len(dec) != plainBlock.Size {
+			return fmt.Errorf("plaintext block %d size mismatch, actual %d != expected %d", i, len(dec), plainBlock.Size)
+		}
+
+		// Verify the hash against the plaintext block info
+		if !scanner.Validate(dec, plainBlock.Hash, 0) {
+			// The block decrypted correctly but fails the hash check. This
+			// is odd and unexpected, but it it's still a valid block from
+			// the source. The file might have changed while we pulled it?
+			err := fmt.Errorf("plaintext block %d (%d bytes) failed validation after decryption", i, plainBlock.Size)
+			if c.Continue {
+				log.Printf("Warning: %s: %s: %v", encFi.Name, plainFi.Name, err)
+			} else {
+				return err
+			}
+		}
+
+		// Write it to the destination, unless we're just verifying.
+		if dst != nil {
+			if _, err := dst.WriteAt(dec, plainBlock.Offset); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// loadEncryptedFileInfo loads the encrypted FileInfo trailer from a file on
+// disk.
+func loadEncryptedFileInfo(fd fs.File) (*protocol.FileInfo, error) {
+	// Seek to the size of the trailer block
+	if _, err := fd.Seek(-4, io.SeekEnd); err != nil {
+		return nil, err
+	}
+	var bs [4]byte
+	if _, err := io.ReadFull(fd, bs[:]); err != nil {
+		return nil, err
+	}
+	size := int64(binary.BigEndian.Uint32(bs[:]))
+
+	// Seek to the start of the trailer
+	if _, err := fd.Seek(-(4 + size), io.SeekEnd); err != nil {
+		return nil, err
+	}
+	trailer := make([]byte, size)
+	if _, err := io.ReadFull(fd, trailer); err != nil {
+		return nil, err
+	}
+
+	var encFi protocol.FileInfo
+	if err := encFi.Unmarshal(trailer); err != nil {
+		return nil, err
+	}
+
+	return &encFi, nil
+}

cmd/syncthing/generate/generate.go

@@ -0,0 +1,136 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Package generate implements the `syncthing generate` subcommand.
+package generate
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/fs"
+	"github.com/syncthing/syncthing/lib/locations"
+	"github.com/syncthing/syncthing/lib/logger"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/syncthing"
+)
+
+type CLI struct {
+	cmdutil.CommonOptions
+	GUIUser     string `placeholder:"STRING" help:"Specify new GUI authentication user name"`
+	GUIPassword string `placeholder:"STRING" help:"Specify new GUI authentication password (use - to read from standard input)"`
+}
+
+func (c *CLI) Run(l logger.Logger) error {
+	if c.HideConsole {
+		osutil.HideConsole()
+	}
+
+	if c.HomeDir != "" {
+		if c.ConfDir != "" {
+			return errors.New("--home must not be used together with --config")
+		}
+		c.ConfDir = c.HomeDir
+	}
+	if c.ConfDir == "" {
+		c.ConfDir = locations.GetBaseDir(locations.ConfigBaseDir)
+	}
+
+	// Support reading the password from a pipe or similar
+	if c.GUIPassword == "-" {
+		reader := bufio.NewReader(os.Stdin)
+		password, _, err := reader.ReadLine()
+		if err != nil {
+			return fmt.Errorf("failed reading GUI password: %w", err)
+		}
+		c.GUIPassword = string(password)
+	}
+
+	if err := Generate(l, c.ConfDir, c.GUIUser, c.GUIPassword, c.NoDefaultFolder, c.SkipPortProbing); err != nil {
+		return fmt.Errorf("failed to generate config and keys: %w", err)
+	}
+	return nil
+}
+
+func Generate(l logger.Logger, confDir, guiUser, guiPassword string, noDefaultFolder, skipPortProbing bool) error {
+	dir, err := fs.ExpandTilde(confDir)
+	if err != nil {
+		return err
+	}
+
+	if err := syncthing.EnsureDir(dir, 0700); err != nil {
+		return err
+	}
+	locations.SetBaseDir(locations.ConfigBaseDir, dir)
+
+	var myID protocol.DeviceID
+	certFile, keyFile := locations.Get(locations.CertFile), locations.Get(locations.KeyFile)
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err == nil {
+		l.Warnln("Key exists; will not overwrite.")
+	} else {
+		cert, err = syncthing.GenerateCertificate(certFile, keyFile)
+		if err != nil {
+			return fmt.Errorf("create certificate: %w", err)
+		}
+	}
+	myID = protocol.NewDeviceID(cert.Certificate[0])
+	l.Infoln("Device ID:", myID)
+
+	cfgFile := locations.Get(locations.ConfigFile)
+	cfg, _, err := config.Load(cfgFile, myID, events.NoopLogger)
+	if fs.IsNotExist(err) {
+		if cfg, err = syncthing.DefaultConfig(cfgFile, myID, events.NoopLogger, noDefaultFolder, skipPortProbing); err != nil {
+			return fmt.Errorf("create config: %w", err)
+		}
+	} else if err != nil {
+		return fmt.Errorf("load config: %w", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go cfg.Serve(ctx)
+	defer cancel()
+
+	var updateErr error
+	waiter, err := cfg.Modify(func(cfg *config.Configuration) {
+		updateErr = updateGUIAuthentication(l, &cfg.GUI, guiUser, guiPassword)
+	})
+	if err != nil {
+		return fmt.Errorf("modify config: %w", err)
+	}
+
+	waiter.Wait()
+	if updateErr != nil {
+		return updateErr
+	}
+	if err := cfg.Save(); err != nil {
+		return fmt.Errorf("save config: %w", err)
+	}
+	return nil
+}
+
+func updateGUIAuthentication(l logger.Logger, guiCfg *config.GUIConfiguration, guiUser, guiPassword string) error {
+	if guiUser != "" && guiCfg.User != guiUser {
+		guiCfg.User = guiUser
+		l.Infoln("Updated GUI authentication user name:", guiUser)
+	}
+
+	if guiPassword != "" && guiCfg.Password != guiPassword {
+		if err := guiCfg.HashAndSetPassword(guiPassword); err != nil {
+			return fmt.Errorf("failed to set GUI authentication password: %w", err)
+		}
+		l.Infoln("Updated GUI authentication password.")
+	}
+	return nil
+}

cmd/syncthing/heapprof.go

@@ -11,24 +11,17 @@ import (
 	"os"
 	"runtime"
 	"runtime/pprof"
-	"strconv"
 	"syscall"
 	"time"
 )
 
-func init() {
+func startHeapProfiler() {
-	if innerProcess && os.Getenv("STHEAPPROFILE") != "" {
+	l.Debugln("Starting heap profiling")
-		rate := 1
+	go func() {
-		if i, err := strconv.Atoi(os.Getenv("STHEAPPROFILE")); err == nil {
+		err := saveHeapProfiles(1) // Only returns on error
-			rate = i
+		l.Warnln("Heap profiler failed:", err)
-		}
+		panic("Heap profiler failed")
-		l.Debugln("Starting heap profiling")
+	}()
-		go func() {
-			err := saveHeapProfiles(rate) // Only returns on error
-			l.Warnln("Heap profiler failed:", err)
-			panic("Heap profiler failed")
-		}()
-	}
 }
 
 func saveHeapProfiles(rate int) error {

cmd/syncthing/monitor.go

@@ -15,18 +15,16 @@ import (
 	"os/exec"
 	"os/signal"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"syscall"
 	"time"
 
-	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/osutil"
-	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/svcutil"
 	"github.com/syncthing/syncthing/lib/sync"
-	"github.com/syncthing/syncthing/lib/syncthing"
 )
 
 var (
@@ -36,20 +34,21 @@ var (
 )
 
 const (
-	countRestarts         = 4
+	restartCounts         = 4
-	loopThreshold         = 60 * time.Second
+	restartPause          = 1 * time.Second
+	restartLoopThreshold  = 60 * time.Second
 	logFileAutoCloseDelay = 5 * time.Second
 	logFileMaxOpenTime    = time.Minute
 	panicUploadMaxWait    = 30 * time.Second
 	panicUploadNoticeWait = 10 * time.Second
 )
 
-func monitorMain(runtimeOptions RuntimeOptions) {
+func monitorMain(options serveOptions) {
 	l.SetPrefix("[monitor] ")
 
 	var dst io.Writer = os.Stdout
 
-	logFile := runtimeOptions.logFile
+	logFile := locations.Get(locations.LogFile)
 	if logFile != "-" {
 		if expanded, err := fs.ExpandTilde(logFile); err == nil {
 			logFile = expanded
@@ -59,15 +58,15 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 		open := func(name string) (io.WriteCloser, error) {
 			return newAutoclosedFile(name, logFileAutoCloseDelay, logFileMaxOpenTime)
 		}
-		if runtimeOptions.logMaxSize > 0 {
+		if options.LogMaxSize > 0 {
-			fileDst, err = newRotatedFile(logFile, open, int64(runtimeOptions.logMaxSize), runtimeOptions.logMaxFiles)
+			fileDst, err = newRotatedFile(logFile, open, int64(options.LogMaxSize), options.LogMaxFiles)
 		} else {
 			fileDst, err = open(logFile)
 		}
 		if err != nil {
 			l.Warnln("Failed to setup logging to file, proceeding with logging to stdout only:", err)
 		} else {
-			if runtime.GOOS == "windows" {
+			if build.IsWindows {
 				// Translate line breaks to Windows standard
 				fileDst = osutil.ReplacingWriter{
 					Writer: fileDst,
@@ -84,7 +83,12 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 	}
 
 	args := os.Args
-	var restarts [countRestarts]time.Time
+	binary, err := getBinary(args[0])
+	if err != nil {
+		l.Warnln("Error starting the main Syncthing process:", err)
+		panic("Error starting the main Syncthing process")
+	}
+	var restarts [restartCounts]time.Time
 
 	stopSign := make(chan os.Signal, 1)
 	signal.Notify(stopSign, os.Interrupt, sigTerm)
@@ -97,15 +101,15 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 	for {
 		maybeReportPanics()
 
-		if t := time.Since(restarts[0]); t < loopThreshold {
+		if t := time.Since(restarts[0]); t < restartLoopThreshold {
-			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
+			l.Warnf("%d restarts in %v; not retrying further", restartCounts, t)
-			os.Exit(syncthing.ExitError.AsInt())
+			os.Exit(svcutil.ExitError.AsInt())
 		}
 
 		copy(restarts[0:], restarts[1:])
 		restarts[len(restarts)-1] = time.Now()
 
-		cmd := exec.Command(args[0], args[1:]...)
+		cmd := exec.Command(binary, args[1:]...)
 		cmd.Env = childEnv
 
 		stderr, err := cmd.StderrPipe()
@@ -169,31 +173,31 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 
 		if err == nil {
 			// Successful exit indicates an intentional shutdown
-			os.Exit(syncthing.ExitSuccess.AsInt())
+			os.Exit(svcutil.ExitSuccess.AsInt())
 		}
 
 		if exiterr, ok := err.(*exec.ExitError); ok {
 			exitCode := exiterr.ExitCode()
-			if stopped || runtimeOptions.noRestart {
+			if stopped || options.NoRestart {
 				os.Exit(exitCode)
 			}
-			if exitCode == syncthing.ExitUpgrade.AsInt() {
+			if exitCode == svcutil.ExitUpgrade.AsInt() {
 				// Restart the monitor process to release the .old
 				// binary as part of the upgrade process.
 				l.Infoln("Restarting monitor...")
-				if err = restartMonitor(args); err != nil {
+				if err = restartMonitor(binary, args); err != nil {
 					l.Warnln("Restart:", err)
 				}
 				os.Exit(exitCode)
 			}
 		}
 
-		if runtimeOptions.noRestart {
+		if options.NoRestart {
-			os.Exit(syncthing.ExitError.AsInt())
+			os.Exit(svcutil.ExitError.AsInt())
 		}
 
 		l.Infoln("Syncthing exited:", err)
-		time.Sleep(1 * time.Second)
+		time.Sleep(restartPause)
 
 		if first {
 			// Let the next child process know that this is not the first time
@@ -204,6 +208,19 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 	}
 }
 
+func getBinary(args0 string) (string, error) {
+	e, err := os.Executable()
+	if err == nil {
+		return e, nil
+	}
+	// Check if args0 cuts it
+	e, lerr := exec.LookPath(args0)
+	if lerr == nil {
+		return e, nil
+	}
+	return "", err
+}
+
 func copyStderr(stderr io.Reader, dst io.Writer) {
 	br := bufio.NewReader(stderr)
 
@@ -256,7 +273,7 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
 * This crash usually occurs due to one of the following reasons:                *
 *  - Syncthing being stopped abruptly (killed/loss of power)                    *
 *  - Bad hardware (memory/disk issues)                                          *
-*  - Software that affects disk writes (SSD caching software and simillar)      *
+*  - Software that affects disk writes (SSD caching software and similar)       *
 *                                                                               *
 * Please see the following URL for instructions on how to recover:              *
 *   https://docs.syncthing.net/users/faq.html#my-syncthing-database-is-corrupt  *
@@ -310,40 +327,30 @@ func copyStdout(stdout io.Reader, dst io.Writer) {
 	}
 }
 
-func restartMonitor(args []string) error {
+func restartMonitor(binary string, args []string) error {
 	// Set the STRESTART environment variable to indicate to the next
 	// process that this is a restart and not initial start. This prevents
 	// opening the browser on startup.
 	os.Setenv("STRESTART", "yes")
 
-	if runtime.GOOS != "windows" {
+	if !build.IsWindows {
 		// syscall.Exec is the cleanest way to restart on Unixes as it
 		// replaces the current process with the new one, keeping the pid and
 		// controlling terminal and so on
-		return restartMonitorUnix(args)
+		return restartMonitorUnix(binary, args)
 	}
 
 	// but it isn't supported on Windows, so there we start a normal
 	// exec.Command and return.
-	return restartMonitorWindows(args)
+	return restartMonitorWindows(binary, args)
 }
 
-func restartMonitorUnix(args []string) error {
+func restartMonitorUnix(binary string, args []string) error {
-	if !strings.ContainsRune(args[0], os.PathSeparator) {
-		// The path to the binary doesn't contain a slash, so it should be
-		// found in $PATH.
-		binary, err := exec.LookPath(args[0])
-		if err != nil {
-			return err
-		}
-		args[0] = binary
-	}
-
 	return syscall.Exec(args[0], args, os.Environ())
 }
 
-func restartMonitorWindows(args []string) error {
+func restartMonitorWindows(binary string, args []string) error {
-	cmd := exec.Command(args[0], args[1:]...)
+	cmd := exec.Command(binary, args[1:]...)
 	// Retain the standard streams
 	cmd.Stderr = os.Stderr
 	cmd.Stdout = os.Stdout
@@ -563,7 +570,7 @@ func childEnv() []string {
 // panicUploadMaxWait uploading panics...
 func maybeReportPanics() {
 	// Try to get a config to see if/where panics should be reported.
-	cfg, err := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger)
+	cfg, err := loadOrDefaultConfig()
 	if err != nil {
 		l.Warnln("Couldn't load config; not reporting crash")
 		return

cmd/syncthing/monitor_test.go

@@ -8,7 +8,6 @@ package main
 
 import (
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
@@ -18,14 +17,17 @@ import (
 func TestRotatedFile(t *testing.T) {
 	// Verify that log rotation happens.
 
-	dir, err := ioutil.TempDir("", "syncthing")
+	dir := t.TempDir()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(dir)
 
 	open := func(name string) (io.WriteCloser, error) {
-		return os.Create(name)
+		f, err := os.Create(name)
+		t.Cleanup(func() {
+			if f != nil {
+				_ = f.Close()
+			}
+		})
+
+		return f, err
 	}
 
 	logName := filepath.Join(dir, "log.txt")
@@ -179,7 +181,7 @@ func TestAutoClosedFile(t *testing.T) {
 	}
 
 	// The file should have both writes in it.
-	bs, err := ioutil.ReadFile(file)
+	bs, err := os.ReadFile(file)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -199,7 +201,7 @@ func TestAutoClosedFile(t *testing.T) {
 	}
 
 	// It should now contain three writes, as the file is always opened for appending
-	bs, err = ioutil.ReadFile(file)
+	bs, err = os.ReadFile(file)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/syncthing/openurl_unix.go

@@ -4,26 +4,25 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
+//go:build !windows
 // +build !windows
 
 package main
 
 import (
 	"os/exec"
-	"runtime"
 	"syscall"
+
+	"github.com/syncthing/syncthing/lib/build"
 )
 
 func openURL(url string) error {
-	switch runtime.GOOS {
+	if build.IsDarwin {
-	case "darwin":
 		return exec.Command("open", url).Run()
-
-	default:
-		cmd := exec.Command("xdg-open", url)
-		cmd.SysProcAttr = &syscall.SysProcAttr{
-			Setpgid: true,
-		}
-		return cmd.Run()
 	}
+	cmd := exec.Command("xdg-open", url)
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+	return cmd.Run()
 }

cmd/syncthing/openurl_windows.go

@@ -4,6 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
+//go:build windows
 // +build windows
 
 package main

cmd/syncthing/perfstats_unix.go

@@ -4,6 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
+//go:build !solaris && !windows
 // +build !solaris,!windows
 
 package main
@@ -18,10 +19,8 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func init() {
+func startPerfStats() {
-	if innerProcess && os.Getenv("STPERFSTATS") != "" {
+	go savePerfStats(fmt.Sprintf("perfstats-%d.csv", syscall.Getpid()))
-		go savePerfStats(fmt.Sprintf("perfstats-%d.csv", syscall.Getpid()))
-	}
 }
 
 func savePerfStats(file string) {

cmd/syncthing/perfstats_unsupported.go

@@ -1,13 +1,13 @@
-// Copyright (C) 2017 The Syncthing Authors.
+// Copyright (C) 2021 The Syncthing Authors.
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//+build noupgrade
+//go:build solaris || windows
+// +build solaris windows
 
-package build
+package main
 
-func init() {
+func startPerfStats() {
-	Tags = append(Tags, "noupgrade")
 }

cmd/syncthing/traceback.go

@@ -4,7 +4,8 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//+build go1.7
+//go:build go1.7
+// +build go1.7
 
 package main
 

cmd/syncthing/usage.go

@@ -1,57 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"flag"
-	"fmt"
-	"io"
-	"text/tabwriter"
-)
-
-func optionTable(w io.Writer, rows [][]string) {
-	tw := tabwriter.NewWriter(w, 2, 4, 2, ' ', 0)
-	for _, row := range rows {
-		for i, cell := range row {
-			if i > 0 {
-				tw.Write([]byte("\t"))
-			}
-			tw.Write([]byte(cell))
-		}
-		tw.Write([]byte("\n"))
-	}
-	tw.Flush()
-}
-
-func usageFor(fs *flag.FlagSet, usage string, extra string) func() {
-	return func() {
-		var b bytes.Buffer
-		b.WriteString("Usage:\n  " + usage + "\n")
-
-		var options [][]string
-		fs.VisitAll(func(f *flag.Flag) {
-			var opt = "  -" + f.Name
-
-			if f.DefValue != "false" {
-				opt += "=" + fmt.Sprintf(`"%s"`, f.DefValue)
-			}
-			options = append(options, []string{opt, f.Usage})
-		})
-
-		if len(options) > 0 {
-			b.WriteString("\nOptions:\n")
-			optionTable(&b, options)
-		}
-
-		fmt.Println(b.String())
-
-		if len(extra) > 0 {
-			fmt.Println(extra)
-		}
-	}
-}

cmd/uraggregate/main.go

@@ -47,32 +47,25 @@ func main() {
 func runAggregation(db *sql.DB) {
 	since := maxIndexedDay(db, "VersionSummary")
 	log.Println("Aggregating VersionSummary data since", since)
-	rows, err := aggregateVersionSummary(db, since)
+	rows, err := aggregateVersionSummary(db, since.Add(24*time.Hour))
 	if err != nil {
-		log.Fatalln("aggregate:", err)
+		log.Println("aggregate:", err)
 	}
 	log.Println("Inserted", rows, "rows")
 
-	log.Println("Aggregating UserMovement data")
-	rows, err = aggregateUserMovement(db)
-	if err != nil {
-		log.Fatalln("aggregate:", err)
-	}
-	log.Println("Inserted", rows, "rows")
-
-	log.Println("Aggregating Performance data")
 	since = maxIndexedDay(db, "Performance")
-	rows, err = aggregatePerformance(db, since)
+	log.Println("Aggregating Performance data since", since)
+	rows, err = aggregatePerformance(db, since.Add(24*time.Hour))
 	if err != nil {
-		log.Fatalln("aggregate:", err)
+		log.Println("aggregate:", err)
 	}
 	log.Println("Inserted", rows, "rows")
 
-	log.Println("Aggregating BlockStats data")
 	since = maxIndexedDay(db, "BlockStats")
-	rows, err = aggregateBlockStats(db, since)
+	log.Println("Aggregating BlockStats data since", since)
+	rows, err = aggregateBlockStats(db, since.Add(24*time.Hour))
 	if err != nil {
-		log.Fatalln("aggregate:", err)
+		log.Println("aggregate:", err)
 	}
 	log.Println("Inserted", rows, "rows")
 }
@@ -119,13 +112,13 @@ func setupDB(db *sql.DB) error {
 	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS BlockStats (
 		Day TIMESTAMP NOT NULL,
 		Reports INTEGER NOT NULL,
-		Total INTEGER NOT NULL,
+		Total BIGINT NOT NULL,
-		Renamed INTEGER NOT NULL,
+		Renamed BIGINT NOT NULL,
-		Reused INTEGER NOT NULL,
+		Reused BIGINT NOT NULL,
-		Pulled INTEGER NOT NULL,
+		Pulled BIGINT NOT NULL,
-		CopyOrigin INTEGER NOT NULL,
+		CopyOrigin BIGINT NOT NULL,
-		CopyOriginShifted INTEGER NOT NULL,
+		CopyOriginShifted BIGINT NOT NULL,
-		CopyElsewhere INTEGER NOT NULL
+		CopyElsewhere BIGINT NOT NULL
 	)`)
 	if err != nil {
 		return err
@@ -135,35 +128,35 @@ func setupDB(db *sql.DB) error {
 
 	row := db.QueryRow(`SELECT 'UniqueDayVersionIndex'::regclass`)
 	if err := row.Scan(&t); err != nil {
-		_, err = db.Exec(`CREATE UNIQUE INDEX UniqueDayVersionIndex ON VersionSummary (Day, Version)`)
+		_, _ = db.Exec(`CREATE UNIQUE INDEX UniqueDayVersionIndex ON VersionSummary (Day, Version)`)
 	}
 
 	row = db.QueryRow(`SELECT 'VersionDayIndex'::regclass`)
 	if err := row.Scan(&t); err != nil {
-		_, err = db.Exec(`CREATE INDEX VersionDayIndex ON VersionSummary (Day)`)
+		_, _ = db.Exec(`CREATE INDEX VersionDayIndex ON VersionSummary (Day)`)
 	}
 
 	row = db.QueryRow(`SELECT 'MovementDayIndex'::regclass`)
 	if err := row.Scan(&t); err != nil {
-		_, err = db.Exec(`CREATE INDEX MovementDayIndex ON UserMovement (Day)`)
+		_, _ = db.Exec(`CREATE INDEX MovementDayIndex ON UserMovement (Day)`)
 	}
 
 	row = db.QueryRow(`SELECT 'PerformanceDayIndex'::regclass`)
 	if err := row.Scan(&t); err != nil {
-		_, err = db.Exec(`CREATE INDEX PerformanceDayIndex ON Performance (Day)`)
+		_, _ = db.Exec(`CREATE INDEX PerformanceDayIndex ON Performance (Day)`)
 	}
 
 	row = db.QueryRow(`SELECT 'BlockStatsDayIndex'::regclass`)
 	if err := row.Scan(&t); err != nil {
-		_, err = db.Exec(`CREATE INDEX BlockStatsDayIndex ON BlockStats (Day)`)
+		_, _ = db.Exec(`CREATE INDEX BlockStatsDayIndex ON BlockStats (Day)`)
 	}
 
-	return err
+	return nil
 }
 
 func maxIndexedDay(db *sql.DB, table string) time.Time {
 	var t time.Time
-	row := db.QueryRow("SELECT MAX(Day) FROM " + table)
+	row := db.QueryRow("SELECT MAX(DATE_TRUNC('day', Day)) FROM " + table)
 	err := row.Scan(&t)
 	if err != nil {
 		return time.Time{}
@@ -175,13 +168,13 @@ func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) {
 	res, err := db.Exec(`INSERT INTO VersionSummary (
 	SELECT
 		DATE_TRUNC('day', Received) AS Day,
-		SUBSTRING(Version FROM '^v\d.\d+') AS Ver,
+		SUBSTRING(Report->>'version' FROM '^v\d.\d+') AS Ver,
 		COUNT(*) AS Count
-		FROM Reports
+		FROM ReportsJson
 		WHERE
-			DATE_TRUNC('day', Received) > $1
+			Received > $1
-			AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
+			AND Received < DATE_TRUNC('day', NOW())
-			AND Version like 'v_.%'
+			AND Report->>'version' like 'v_.%'
 		GROUP BY Day, Ver
 		);
 	`, since)
@@ -195,11 +188,12 @@ func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) {
 func aggregateUserMovement(db *sql.DB) (int64, error) {
 	rows, err := db.Query(`SELECT
 		DATE_TRUNC('day', Received) AS Day,
-		UniqueID
+		Report->>'uniqueID'
-		FROM Reports
+		FROM ReportsJson
 		WHERE
-			DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
+			Report->>'uniqueID' IS NOT NULL
-			AND Version like 'v_.%'
+			AND Received < DATE_TRUNC('day', NOW())
+			AND Report->>'version' like 'v_.%'
 		ORDER BY Day
 	`)
 	if err != nil {
@@ -276,16 +270,18 @@ func aggregatePerformance(db *sql.DB, since time.Time) (int64, error) {
 	res, err := db.Exec(`INSERT INTO Performance (
 	SELECT
 		DATE_TRUNC('day', Received) AS Day,
-		AVG(TotFiles) As TotFiles,
+		AVG((Report->>'totFiles')::numeric) As TotFiles,
-		AVG(TotMiB) As TotMiB,
+		AVG((Report->>'totMiB')::numeric) As TotMiB,
-		AVG(SHA256Perf) As SHA256Perf,
+		AVG((Report->>'sha256Perf')::numeric) As SHA256Perf,
-		AVG(MemorySize) As MemorySize,
+		AVG((Report->>'memorySize')::numeric) As MemorySize,
-		AVG(MemoryUsageMiB) As MemoryUsageMiB
+		AVG((Report->>'memoryUsageMiB')::numeric) As MemoryUsageMiB
-		FROM Reports
+		FROM ReportsJson
 		WHERE
-			DATE_TRUNC('day', Received) > $1
+			Received > $1
-			AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
+			AND Received < DATE_TRUNC('day', NOW())
-			AND Version like 'v_.%'
+			AND Report->>'version' like 'v_.%'
+			/* Some custom implementation reported bytes when we expect megabytes, cap at petabyte */
+			AND (Report->>'memorySize')::numeric < 1073741824
 		GROUP BY Day
 		);
 	`, since)
@@ -303,22 +299,22 @@ func aggregateBlockStats(db *sql.DB, since time.Time) (int64, error) {
 	SELECT
 		DATE_TRUNC('day', Received) AS Day,
 		COUNT(1) As Reports,
-		SUM(BlocksTotal) AS Total,
+		SUM((Report->'blockStats'->>'total')::numeric)::bigint AS Total,
-		SUM(BlocksRenamed) AS Renamed,
+		SUM((Report->'blockStats'->>'renamed')::numeric)::bigint AS Renamed,
-		SUM(BlocksReused) AS Reused,
+		SUM((Report->'blockStats'->>'reused')::numeric)::bigint AS Reused,
-		SUM(BlocksPulled) AS Pulled,
+		SUM((Report->'blockStats'->>'pulled')::numeric)::bigint AS Pulled,
-		SUM(BlocksCopyOrigin) AS CopyOrigin,
+		SUM((Report->'blockStats'->>'copyOrigin')::numeric)::bigint AS CopyOrigin,
-		SUM(BlocksCopyOriginShifted) AS CopyOriginShifted,
+		SUM((Report->'blockStats'->>'copyOriginShifted')::numeric)::bigint AS CopyOriginShifted,
-		SUM(BlocksCopyElsewhere) AS CopyElsewhere
+		SUM((Report->'blockStats'->>'copyElsewhere')::numeric)::bigint AS CopyElsewhere
-		FROM Reports
+		FROM ReportsJson
 		WHERE
-			DATE_TRUNC('day', Received) > $1
+			Received > $1
-			AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
+			AND Received < DATE_TRUNC('day', NOW())
-			AND ReportVersion = 3
+			AND (Report->>'urVersion')::numeric >= 3
-			AND Version like 'v_.%'
+			AND Report->>'version' like 'v_.%'
-			AND Version NOT LIKE 'v0.14.40%'
+			AND Report->>'version' NOT LIKE 'v0.14.40%'
-			AND Version NOT LIKE 'v0.14.39%'
+			AND Report->>'version' NOT LIKE 'v0.14.39%'
-			AND Version NOT LIKE 'v0.14.38%'
+			AND Report->>'version' NOT LIKE 'v0.14.38%'
 		GROUP BY Day
 	);
 	`, since)

cmd/ursrv/analytics.go

@@ -114,6 +114,23 @@ func statsForInts(data []int) [4]float64 {
 	return res
 }
 
+func statsForInt64s(data []int64) [4]float64 {
+	var res [4]float64
+	if len(data) == 0 {
+		return res
+	}
+
+	sort.Slice(data, func(a, b int) bool {
+		return data[a] < data[b]
+	})
+
+	res[0] = float64(data[int(float64(len(data))*0.05)])
+	res[1] = float64(data[len(data)/2])
+	res[2] = float64(data[int(float64(len(data))*0.95)])
+	res[3] = float64(data[len(data)-1])
+	return res
+}
+
 func statsForFloats(data []float64) [4]float64 {
 	var res [4]float64
 	if len(data) == 0 {