build: Use Go 1.22.3 at minimum

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+/AUTHORS    @calmh
+/*.md       @calmh

.github/ISSUE_TEMPLATE/01-feature.yml

@@ -1,7 +1,6 @@
 name: Feature request
 description: File a new feature request
 labels: ["enhancement", "needs-triage"]
-type: Feature
 body:
 
   - type: textarea

.github/ISSUE_TEMPLATE/02-bug.yml

@@ -1,7 +1,6 @@
 name: Bug report
 description: If you're actually looking for support instead, see "I need help / I have a question".
 labels: ["bug", "needs-triage"]
-type: Bug
 body:
   - type: markdown
     attributes:

.github/dependabot.yml

@@ -3,11 +3,11 @@ updates:
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: monthly
+      interval: weekly
     open-pull-requests-limit: 10
 
   - package-ecosystem: "gomod"
     directory: "/"
     schedule:
-      interval: monthly
+      interval: weekly
     open-pull-requests-limit: 10

.github/labeler.yml

@@ -1,23 +0,0 @@
-version: 1
-labels:
-
-- label: enhancement
-  title: ^feat\b
-
-- label: bug
-  title: ^fix\b
-
-- label: documentation
-  title: ^docs\b
-
-- label: chore
-  title: ^chore\b
-
-- label: chore
-  title: ^refactor\b
-
-- label: build
-  title: ^build\b
-
-- label: dependencies
-  title: ^build\(deps\)\b

.github/regsync.yml

@@ -1,52 +0,0 @@
-  version: 1
-  creds:
-    - registry: docker.io
-      user: "{{env \"DOCKERHUB_USERNAME\"}}"
-      pass: "{{env \"DOCKERHUB_TOKEN\"}}"
-
-  defaults:
-    ratelimit:
-      min: 100
-      retry: 1m
-    parallel: 4
-
-  sync:
-
-    - source: ghcr.io/syncthing/syncthing
-      target: docker.io/syncthing/syncthing
-      type: repository
-      tags:
-        allow:
-          - latest
-          - rc
-          - edge
-          - \d+
-          - \d+\.\d+
-          - \d+\.\d+\.\d+
-          - \d+\.\d+\.\d+-rc\.\d+
-
-    - source: ghcr.io/syncthing/relaysrv
-      target: docker.io/syncthing/relaysrv
-      type: repository
-      tags:
-        allow:
-          - latest
-          - rc
-          - edge
-          - \d+
-          - \d+\.\d+
-          - \d+\.\d+\.\d+
-          - \d+\.\d+\.\d+-rc\.\d+
-
-    - source: ghcr.io/syncthing/discosrv
-      target: docker.io/syncthing/discosrv
-      type: repository
-      tags:
-        allow:
-          - latest
-          - rc
-          - edge
-          - \d+
-          - \d+\.\d+
-          - \d+\.\d+\.\d+
-          - \d+\.\d+\.\d+-rc\.\d+

.github/release.yml

@@ -1,17 +0,0 @@
-changelog:
-  exclude:
-    labels:
-      - dependencies
-
-  categories:
-    - title: Fixes
-      labels:
-        - bug
-
-    - title: Features
-      labels:
-        - enhancement
-
-    - title: Other
-      labels:
-        - '*'

.github/workflows/build-infra-dockers.yaml

@@ -7,15 +7,11 @@ on:
       - infra-*
 
 env:
-  GO_VERSION: "~1.25.0"
+  GO_VERSION: "~1.22.3"
   CGO_ENABLED: "0"
   BUILD_USER: docker
   BUILD_HOST: github.syncthing.net
 
-permissions:
-  contents: read
-  packages: write
-
 jobs:
   docker-syncthing:
     name: Build and push Docker images
@@ -30,11 +26,11 @@ jobs:
           - stupgrades
           - ursrv
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
           check-latest: true
@@ -45,13 +41,6 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Login to GHCR
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Build binaries
         run: |
           for arch in arm64 amd64; do
@@ -64,13 +53,13 @@ jobs:
 
       - name: Set Docker tags (all branches)
         run: |
-          tags=docker.io/syncthing/${{ matrix.pkg }}:${{ github.sha }},ghcr.io/syncthing/infra/${{ matrix.pkg }}:${{ github.sha }}
+          tags=syncthing/${{ matrix.pkg }}:${{ github.sha }}
           echo "TAGS=$tags" >> $GITHUB_ENV
 
       - name: Set Docker tags (latest)
         if: github.ref == 'refs/heads/infrastructure'
         run: |
-          tags=docker.io/syncthing/${{ matrix.pkg }}:latest,ghcr.io/syncthing/infra/${{ matrix.pkg }}:latest,${{ env.TAGS }}
+          tags=syncthing/${{ matrix.pkg }}:latest,${{ env.TAGS }}
           echo "TAGS=$tags" >> $GITHUB_ENV
 
       - name: Build and push

.github/workflows/build-nightly.yaml

@@ -1,18 +0,0 @@
-name: Build Syncthing (Nightly)
-
-on:
-  schedule:
-    # Run nightly build at 05:00 UTC
-    - cron: '00 05 * * *'
-  workflow_dispatch:
-
-permissions:
-  contents: write
-  packages: write
-
-jobs:
-  build-syncthing:
-    uses: ./.github/workflows/build-syncthing.yaml
-    # if we only want nightlies to run for specific users:
-    # if: contains(fromJSON('["syncthing", "calmh"]'), github.repository_owner)
-    secrets: inherit

.github/workflows/mirrors.yaml

@@ -1,18 +0,0 @@
-name: Mirrors
-
-on: [push, delete]
-
-jobs:
-  codeberg:
-    name: Mirror to Codeberg
-    if: github.repository_owner == 'syncthing'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: yesolutions/mirror-action@master
-        with:
-          REMOTE: ssh://git@codeberg.org/${{ github.repository }}.git
-          GIT_SSH_PRIVATE_KEY: ${{ secrets.CODEBERG_PUSH_KEY }}
-          GIT_SSH_NO_VERIFY_HOST: "true"

.github/workflows/org-members.yaml

@@ -1,20 +0,0 @@
-name: Org membership recommendations
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: '0 0 1 * *'
-
-jobs:
-
-  run-recommendation:
-    runs-on: ubuntu-latest
-    name: Check for a recommendation
-    steps:
-
-      - uses: docker://ghcr.io/calmh/github-org-members:latest
-        env:
-          GITHUB_ORGANISATION: syncthing
-          GITHUB_TOKEN: ${{ secrets.GOM_GITHUB_TOKEN }}
-          GOM_IGNORE_USERS: ${{ secrets.GOM_IGNORE_USERS }}
-          GOM_ALSO_REPOS: ${{ secrets.GOM_ALSO_REPOS }}

.github/workflows/pr-metadata.yaml

@@ -1,27 +0,0 @@
-name: PR metadata
-
-on:
-  pull_request_target:
-    types:
-      - opened
-      - reopened
-      - edited
-      - synchronize
-
-permissions:
-  contents: read
-  pull-requests: write
-
-jobs:
-
-  #
-  # Set labels on PRs, which are then used to categorise release notes
-  #
-
-  labels:
-    name: Set labels
-    runs-on: ubuntu-latest
-    steps:
-    - uses: srvaroa/labeler@v1
-      env:
-        GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/release-syncthing.yaml

@@ -1,60 +0,0 @@
-name: Release Syncthing
-
-on:
-  push:
-    branches:
-      - release
-      - release-rc*
-
-permissions:
-  contents: write
-
-jobs:
-  create-release-tag:
-    name: Create release tag
-    runs-on: ubuntu-latest
-    environment: release
-    steps:
-      - uses: actions/checkout@v5
-        with:
-          fetch-depth: 0
-          ref: ${{ github.ref }} # https://github.com/actions/checkout/issues/882
-          token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
-
-      - uses: actions/setup-go@v6
-        with:
-          go-version: stable
-
-      - name: Determine version to release
-        run: |
-          if [[ "$GITHUB_REF_NAME" == "release" ]] ; then
-            next=$(go run ./script/next-version.go)
-          else
-            next=$(go run ./script/next-version.go --pre)
-          fi
-          echo "NEXT=$next" >> $GITHUB_ENV
-          echo "Next version is $next"
-
-          prev=$(git describe --exclude "*-*" --abbrev=0)
-          echo "PREV=$prev" >> $GITHUB_ENV
-          echo "Previous version is $prev"
-
-      - name: Determine release notes
-        run: |
-          go run ./script/relnotes.go --new-ver "$NEXT" --branch "$GITHUB_REF_NAME" --prev-ver "$PREV" > notes.md
-        env:
-          GITHUB_TOKEN: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
-
-      - name: Create and push tag
-        run: |
-          git config --global user.name 'Syncthing Release Automation'
-          git config --global user.email 'release@syncthing.net'
-          git tag -a -F notes.md --cleanup=whitespace "$NEXT"
-          git push origin "$NEXT"
-
-      - name: Trigger the build
-        uses: benc-uk/workflow-dispatch@v1
-        with:
-          workflow: build-syncthing.yaml
-          ref: refs/tags/${{ env.NEXT }}
-          token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}

.github/workflows/trigger-nightly.yaml

@@ -8,12 +8,11 @@ on:
 jobs:
 
   trigger-nightly:
-    if: github.repository_owner == 'syncthing'
     runs-on: ubuntu-latest
     name: Push to release-nightly to trigger build
     steps:
 
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
         with:
           token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
           fetch-depth: 0

.github/workflows/update-docs-translations.yaml

@@ -10,11 +10,11 @@ jobs:
     runs-on: ubuntu-latest
     name: Update translations and documentation
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v5
         with:
           go-version: stable
       - run: |

.gitignore

@@ -17,4 +17,4 @@ deb
 *.bz2
 /repos
 /proto/scripts/protoc-gen-gosyncthing
-/compat.json
+/gui/next-gen-gui

.golangci.yml

@@ -1,96 +1,26 @@
-version: "2"
+linters-settings:
+  maligned:
+    suggest-new: true
+
 linters:
-  default: all
+  enable-all: true
   disable:
-    - cyclop
+    - goimports
     - depguard
-    - err113
-    - exhaustive
-    - exhaustruct
-    - forbidigo
-    - funcorder
-    - funlen
-    - gochecknoglobals
-    - gochecknoinits
-    - gocognit
-    - goconst
-    - gocyclo
-    - godot
-    - godox
-    - gomoddirectives
-    - inamedparam
-    - interfacebloat
-    - ireturn
     - lll
-    - maintidx
-    - mnd
-    - musttag
-    - nestif
-    - nlreturn
-    - noinlineerr
-    - nonamedreturns
-    - paralleltest
-    - prealloc
-    - predeclared
-    - protogetter
-    - recvcheck
-    - revive
-    - tagalign
-    - tagliatelle
-    - testpackage
-    - usetesting # go 1.24
-    - varnamelen
-    - whitespace
-    - wrapcheck
+    - gochecknoinits
+    - gochecknoglobals
+    - gofmt
+    - scopelint
+    - gocyclo
+    - funlen
     - wsl
-    - wsl_v5
-  exclusions:
-    generated: lax
-    presets:
-      - comments
-      - common-false-positives
-      - legacy
-      - std-error-handling
-    paths:
-      - internal/gen
-      - internal/db/olddb
-      - cmd/dev
-      - repos
-      - third_party$
-      - builtin$
-      - examples$
-      - _test\.go$
-    rules:
-      # relax the slog rules for debug lines, for now
-      - linters: [sloglint]
-        source: Debug
-      # contexts are irrelevant for SQLite
-      - linters: [noctx]
-        text: database/sql
-      # Rollback errors can be ignored
-      - linters: [errcheck]
-        source: Rollback
-      # Embedded fields named in selectors may add clarity
-      - linters: [staticcheck]
-        text: QF1008
-      # Don't necessarily rewrite !(foo || bar) to !foo && !bar
-      - linters: [staticcheck]
-        text: QF1001
-  settings:
-    sloglint:
-      context: "scope"
-      static-msg: true
-      msg-style: capitalized
-      key-naming-case: camel
-formatters:
-  enable:
-    - gofumpt
-  exclusions:
-    generated: lax
-    paths:
-      - internal/gen
-      - cmd/dev
-      - repos
-      - third_party$
-      - builtin$
-      - examples$
+    - gocognit
+    - godox
+
+service:
+  golangci-lint-version: 1.21.x
+  prepare:
+    - rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
+    - GO111MODULE=on go mod vendor
+    - go run build.go assets

.policy.yml

@@ -1,111 +0,0 @@
-# This is the policy-bot configuration for this repository. It controls
-# which approvals are required for any given pull request. The format is
-# described at https://github.com/palantir/policy-bot. The syntax of the
-# policy can be verified by the bot:
-# curl https://pb.syncthing.net/api/validate -X PUT -T .policy.yml
-
-# The policy below is what is required for any pull request.
-policy:
-  approval:
-    - subject is conventional commit
-    - or:
-      - project metadata requires maintainer approval
-      - a maintainer claims responsibility
-    - or:
-      - is approved by a syncthing contributor
-      - is a translation or dependency update by a contributor
-      - is a trivial change by a contributor
-      - a maintainer claims responsibility
-
-  # Additionally, maintainers can disapprove of a PR
-  disapproval:
-    requires:
-      teams:
-        - syncthing/maintainers
-
-# The rules for the policy are described below.
-
-approval_rules:
-
-  # All commits (PRs before squashing) should have a valid conventional
-  # commit type subject.
-  - name: subject is conventional commit
-    requires:
-      conditions:
-        title:
-          matches:
-            - '^(feat|fix|docs|chore|refactor|build): [a-z].+'
-            - '^(feat|fix|docs|chore|refactor|build)\(\w+(, \w+)*\): [a-z].+'
-
-  # Changes to important project metadata and documentation, including this
-  # policy, require signoff by a maintainer
-  - name: project metadata requires maintainer approval
-    if:
-      changed_files:
-        paths:
-          - ^[^/]+\.md
-          - ^\.policy\.yml
-          - ^LICENSE
-    requires:
-      count: 1
-      teams:
-        - syncthing/maintainers
-    options:
-      ignore_update_merges: true
-      allow_non_author_contributor: true
-
-  # Regular pull requests require approval by an active contributor
-  - name: is approved by a syncthing contributor
-    requires:
-      count: 1
-      teams:
-        - syncthing/contributors
-    options:
-      ignore_update_merges: true
-      allow_non_author_contributor: true
-
-  # Changes to some files (translations, dependencies, compatibility) do not
-  # require approval if they were proposed by a contributor and have a
-  # matching commit subject
-  - name: is a translation or dependency update by a contributor
-    if:
-      only_changed_files:
-        paths:
-          - ^gui/default/assets/lang/
-          - ^go\.mod$
-          - ^go\.sum$
-          - ^compat\.yaml$
-      title:
-        matches:
-          - '^chore\(gui\):'
-          - '^build\(deps\):'
-          - '^build\(compat\):'
-      has_author_in:
-        teams:
-          - syncthing/contributors
-
-  # If the change is small and the label "trivial" is added, we accept that
-  # on trust. These PRs can be audited after the fact as appropriate.
-  # Features are not trivial.
-  - name: is a trivial change by a contributor
-    if:
-      modified_lines:
-        total: "< 25"
-      title:
-        not_matches:
-          - '^feat'
-      has_labels:
-        - trivial
-      has_author_in:
-        teams:
-          - syncthing/contributors
-
-  # A member of the maintainers group can take responsibility by adding the
-  # appropriate label.
-  - name: a maintainer claims responsibility
-    if:
-      has_labels:
-        - maintainer-responsibility
-      has_author_in:
-        teams:
-          - syncthing/maintainers

AUTHORS

@@ -13,115 +13,117 @@
 # contents of this file.
 #
 
-Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net> <jborg@coreweave.com>
-Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
-Simon Frei (imsodin) <freisim93@gmail.com>
-Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com> <twilczynski@naver.com>
-Alexander Graf (alex2108) <register-github@alex-graf.de>
-Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
-Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
-André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
-Antony Male (canton7) <antony.male@gmail.com>
-Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
-bt90 <btom1990@googlemail.com>
-Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
-Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
-Emil Lundberg <emil@emlun.se>
-Eric P <eric@kastelo.net>
-Evgeny Kuznetsov <evgeny@kuznetsov.md>
-greatroar <61184462+greatroar@users.noreply.github.com>
-Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
-Lode Hoste (Zillode) <zillode@zillode.be>
-Marcus B Spencer <marcus@marcusspencer.xyz> <marcus@marcusspencer.us>
-Michael Ploujnikov (plouj) <ploujj@gmail.com>
-Ross Smith II (rasa) <ross@smithii.com>
-Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
-Tommy van der Vorst <tommy-github@pixelspark.nl> <tommy@pixelspark.nl>
-Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
+Aaron Bieber (qbit) <qbit@deftly.net>
 Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com> <adam@proactiveservices.co.uk>
 Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
+Alan Pope <alan@popey.com>
+Alberto Donato <albertodonato@users.noreply.github.com>
 Aleksey Vasenev <margtu-fivt@ya.ru>
 Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
-Alex Ionescu <github@ionescu.sh>
 Alex Lindeman <139387+aelindeman@users.noreply.github.com>
 Alex Xu <alex.hello71@gmail.com>
+Alexander Graf (alex2108) <register-github@alex-graf.de>
 Alexander Seiler <seileralex@gmail.com>
 Alexandre Alves <alexandrealvesdb.contact@gmail.com>
+Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
 Aman Gupta <aman@tmm1.net>
+Anatoli Babenia <anatoli@rainforce.org>
+Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
 Andreas Sommer <andreas.sommer87@googlemail.com>
 andresvia <andres.via@gmail.com>
+Andrew Dunham (andrew-d) <andrew@du.nham.ca>
+Andrew Meyer <andrewm.bpi@gmail.com>
 Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
 Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
+André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
 andyleap <andyleap@gmail.com>
 Anjan Momi <anjan@momi.ca>
 Anthony Goeckner <agoeckner@users.noreply.github.com>
 Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
+Antony Male (canton7) <antony.male@gmail.com>
 Anur <anurnomeru@163.com>
 Aranjedeath <Aranjedeath@users.noreply.github.com>
-ardevd <ardevd@users.noreply.github.com>
 Arkadiusz Tymiński <gevleeog@gmail.com>
 Aroun <login@b-vo.fr>
 Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
 Artur Zubilewicz <AkaZecik@users.noreply.github.com>
-Ashish Bhate <bhate.ashish@gmail.com>
+Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
 Aurélien Rainone <476650+arl@users.noreply.github.com>
 BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
 Bart De Vries (mogwa1) <devriesb@gmail.com>
 Beat Reichenbach <44111292+beatreichenbach@users.noreply.github.com>
+Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
+Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
 Ben Shepherd (benshep) <bjashepherd@gmail.com>
 Ben Sidhom (bsidhom) <bsidhom@gmail.com>
 Benedikt Heine (bebehei) <bebe@bebehei.de>
+Benedikt Morbach <benedikt.morbach@googlemail.com>
+Benjamin Nater <17193640+bn4t@users.noreply.github.com>
 Benno Fünfstück <benno.fuenfstueck@gmail.com>
 Benny Ng (tpng) <benny.tpng@gmail.com>
 boomsquared <54829195+boomsquared@users.noreply.github.com>
 Boqin Qin <bobbqqin@bupt.edu.cn>
 Boris Rybalkin <ribalkin@gmail.com>
+Brandon Philips (philips) <brandon@ifup.org>
 Brendan Long (brendanlong) <self@brendanlong.com>
+Brian R. Becker (brbecker) <brbecker@gmail.com>
+bt90 <btom1990@googlemail.com>
+Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
+Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
 Catfriend1 <16361913+Catfriend1@users.noreply.github.com>
 Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
 Cedric Staniewski (xduugu) <cedric@gmx.ca>
+chenrui <rui@meetup.com>
 Chih-Hsuan Yen <yan12125@gmail.com> <1937689+yan12125@users.noreply.github.com>
 Choongkyu <choongkyu.kim+gh@gmail.com> <vapidlyrapid+gh@gmail.com>
 Chris Howie (cdhowie) <me@chrishowie.com>
 Chris Joel (cdata) <chris@scriptolo.gy>
+Chris Tonkinson <chris@masterbran.ch>
 Christian Kujau <ckujau@users.noreply.github.com>
 Christian Prescott <me@christianprescott.com>
 chucic <chucic@seznam.cz>
 cjc7373 <niuchangcun@gmail.com>
 Colin Kennedy (moshen) <moshen.colin@gmail.com>
 Cromefire_ <tim.l@nghorst.net> <26320625+cromefire@users.noreply.github.com>
+cui fliter <imcusg@gmail.com>
 Cyprien Devillez <cypx@users.noreply.github.com>
 d-volution <49024624+d-volution@users.noreply.github.com>
+Dale Visser <dale.visser@live.com>
 Dan <benda.daniel@gmail.com>
 Daniel Barczyk <46358936+DanielBarczyk@users.noreply.github.com>
 Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
+Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
 Daniel Martí (mvdan) <mvdan@mvdan.cc>
 Daniel Padrta <64928366+danpadcz@users.noreply.github.com>
-Daniil Gentili <daniil@daniil.it>
 Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
-dashangcun <907225865@qq.com>
 David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
+deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com>
 DeflateAwning <11021263+DeflateAwning@users.noreply.github.com>
 Denis A. (dva) <denisva@gmail.com>
 Dennis Wilson (snnd) <dw@risu.io>
+dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com> <27856297+dependabot-preview[bot]@users.noreply.github.com>
+dependabot[bot] <dependabot[bot]@users.noreply.github.com> <49699333+dependabot[bot]@users.noreply.github.com>
 derekriemer <derek.riemer@colorado.edu>
 DerRockWolf <50499906+DerRockWolf@users.noreply.github.com>
 desbma <desbma@users.noreply.github.com>
 Devon G. Redekopp <devon@redekopp.com>
+diemade <spamkill@posteo.ch>
 digital <didev@dinid.net>
 Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com>
 Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
-domain <32405309+szu17dmy@users.noreply.github.com>
 Domenic Horner <domenic@tgxn.net>
 Dominik Heidler (asdil12) <dominik@heidler.eu>
 Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
 Elliot Huffman <thelich2@gmail.com>
 Emil Hessman (ceh) <emil@hessman.se>
+Emil Lundberg <emil@emlun.se>
 Eng Zer Jun <engzerjun@gmail.com>
 entity0xfe <109791748+entity0xfe@users.noreply.github.com> <entity0xfe@my.domain>
 Eric Lesiuta <elesiuta@gmail.com>
+Eric P <eric@kastelo.net>
 Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
 Evan Spensley <94762716+0evan@users.noreply.github.com>
+Evgeny Kuznetsov <evgeny@kuznetsov.md>
 Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
 Felix <53702818+f-eliks@users.noreply.github.com>
 Felix Ableitner (Nutomic) <me@nutomic.com>
@@ -135,49 +137,61 @@ ghjklw <malo@jaffre.info>
 Gilli Sigurdsson (gillisig) <gilli@vx.is>
 Gleb Sinyavskiy <zhulik.gleb@gmail.com>
 Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
+greatroar <61184462+greatroar@users.noreply.github.com>
 Greg <gco@jazzhaiku.com>
 guangwu <guoguangwu@magic-shield.com>
 gudvinr <gudvinr@gmail.com>
-Gusted <postmaster@gusted.xyz> <williamzijl7@hotmail.com>
 Han Boetes <han@boetes.org>
 HansK-p <42314815+HansK-p@users.noreply.github.com>
 Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
-Hazem Krimi <me@hazemkrimi.tech>
 Heiko Zuerker (Smiley73) <heiko@zuerker.org>
-Hireworks <129852174+hireworksltd@users.noreply.github.com>
 Hugo Locurcio <hugo.locurcio@hugo.pro>
 Iain Barnett <iainspeed@gmail.com>
 Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
 ignacy123 <ignacy.buczek@onet.pl>
+Ikko Ashimine <eltociear@gmail.com>
+Ilya Brin <464157+ilyabrin@users.noreply.github.com>
 Iskander Sharipov (Alex) <quasilyte@gmail.com>
 Jaakko Hannikainen (jgke) <jgke@jgke.fi>
+Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
 Jack Croft <jccroft1@users.noreply.github.com>
 Jacob <jyundt@gmail.com>
 Jake Peterson (acogdev) <jake@acogdev.com>
+Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net> <jborg@coreweave.com>
 James O'Beirne <wild-github@au92.org>
 James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
+janost <janost@tuta.io>
 Jaroslav Lichtblau <svetlemodry@users.noreply.github.com>
 Jaroslav Malec (dzarda) <dzardacz@gmail.com>
+jaseg <githubaccount@jaseg.net>
 Jaspitta <ste.scarpitta@gmail.com>
+Jauder Ho <jauderho@users.noreply.github.com>
 Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
 Jaya Kumar <jaya.kumar@ict.nl>
 Jeffery To <jeffery.to@gmail.com>
 jelle van der Waa <jelle@vdwaa.nl>
 Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
+Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
+Jesse Lucas <jesse@jesselucas.com>
 Jochen Voss (seehuhn) <voss@seehuhn.de>
+Johan Andersson <j@i19.se>
 Johan Vromans (sciurius) <jvromans@squirrel.nl>
 John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
 Jonas Thelemann <e-mail@jonas-thelemann.de>
 Jonathan <artback@protonmail.com> <jonagn@gmail.com>
+Jonathan Cross <jcross@gmail.com>
+Jonta <359397+Jonta@users.noreply.github.com>
 Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
 jtagcat <git-514635f7@jtag.cat> <git-12dbd862@jtag.cat>
 Julian Lehrhuber <jul13579@users.noreply.github.com>
 Jörg Thalheim <Mic92@users.noreply.github.com>
 Jędrzej Kula <kula.jedrek@gmail.com>
-Kapil Sareen <kapilsareen584@gmail.com>
+K.B.Dharun Krishna <kbdharunkrishna@gmail.com>
+Kalle Laine <pahakalle@protonmail.com>
 Karol Różycki (krozycki) <rozycki.karol@gmail.com>
 Kebin Liu <lkebin@gmail.com>
 Keith Harrison <keithh@protonmail.com>
+Keith Turner <kturner@apache.org>
 Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
 Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
 Kevin Allen (ironmig) <kma1660@gmail.com>
@@ -186,10 +200,13 @@ Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
 klemens <ka7@github.com>
 Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
 kylosus <33132401+kylosus@users.noreply.github.com>
+Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
 Lars Lehtonen <lars.lehtonen@gmail.com>
+Laurent Arnoud <laurent@spkdev.net>
 Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
 Leo Arias (elopio) <yo@elopio.net>
 Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
+Lode Hoste (Zillode) <zillode@zillode.be>
 Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
 LSmithx2 <42276854+lsmithx2@users.noreply.github.com>
 Lukas Lihotzki <lukas@lihotzki.de>
@@ -197,15 +214,16 @@ Luke Hamburg <1992842+luckman212@users.noreply.github.com>
 luzpaz <luzpaz@users.noreply.github.com>
 Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
 Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
-Marcel Meyer <mm.marcelmeyer@gmail.com>
+Marc Pujol (kilburn) <kilburn@la3.org>
 Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
+marco-m <marco.molteni@laposte.net>
 Marcus Legendre <marcus.legendre@gmail.com>
 Mario Majila <mariustshipichik@gmail.com>
 Mark Pulford (mpx) <mark@kyne.com.au>
 Martchus <martchus@gmx.net>
+Martin Polehla <p0l0us@users.noreply.github.com>
 Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
 Mateusz Ż <thedead4fun@live.com>
-mathias4833 <67101597+mathias4833@users.noreply.github.com>
 Matic Potočnik <hairyfotr@gmail.com>
 Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
 Matt Robenolt <matt@ydekproductions.com>
@@ -214,9 +232,13 @@ Maurizio Tomasi <ziotom78@gmail.com>
 Max <github@germancoding.com>
 Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
 MaximAL <almaximal@ya.ru>
+Maxime Thirouin <m@moox.io>
 Maximilian <maxi.rostock@outlook.de> <public@complexvector.space>
+mclang <1721600+mclang@users.noreply.github.com>
 Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
+Michael Ploujnikov (plouj) <ploujj@gmail.com>
 Michael Rienstra <mrienstra@gmail.com>
+Michael Tilli (pyfisch) <pyfisch@gmail.com>
 MichaIng <micha@dietpi.com>
 Migelo <miha@filetki.si>
 Mike Boone <mike@boonedocks.net>
@@ -225,6 +247,7 @@ MikolajTwarog <43782609+MikolajTwarog@users.noreply.github.com>
 Mingxuan Lin <gdlmx@users.noreply.github.com>
 mv1005 <49659413+mv1005@users.noreply.github.com>
 Nate Morrison (nrm21) <natemorrison@gmail.com>
+Naveen <172697+naveensrinivasan@users.noreply.github.com>
 nf <nf@wh3rd.net>
 Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
 Nick Busey <NickBusey@users.noreply.github.com>
@@ -239,13 +262,14 @@ NoLooseEnds <jon.koslung@gmail.com>
 Oliver Freyermuth <o.freyermuth@googlemail.com>
 orangekame3 <miya.org.0309@gmail.com>
 otbutz <tbutz@optitool.de>
+Otiel <Otiel@users.noreply.github.com>
 overkill <22098433+0verk1ll@users.noreply.github.com>
 Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
 Pablo <pbaeyens31+github@gmail.com>
 Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
 Paul Brit <paulbrit44@gmail.com>
-Paul Donald <newtwen+github@gmail.com>
 Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
+Paweł Rozlach <vespian@users.noreply.github.com>
 perewa <cavalcante.ten@gmail.com>
 Peter Badida <KeyWeeUsr@users.noreply.github.com>
 Peter Dave Hello <hsu@peterdavehello.org>
@@ -255,16 +279,18 @@ Phani Rithvij <phanirithvij2000@gmail.com>
 Phil Davis <phil.davis@inf.org>
 Philippe Schommers (filoozoom) <philippe@schommers.be>
 Phill Luby (pluby) <phill.luby@newredo.com>
+Pier Paolo Ramon <ramonpierre@gmail.com>
 Piotr Bejda (piobpl) <piotrb10@gmail.com>
-polyfloyd <polyfloyd@users.noreply.github.com>
-pullmerge <166967364+pullmerge@users.noreply.github.com>
+Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
 Quentin Hibon <qh.public@yahoo.com>
 Rahmi Pruitt <rjpruitt16@gmail.com>
 red_led <red-led@users.noreply.github.com>
+Richard Hartmann <RichiH@users.noreply.github.com>
 Robert Carosi (nov1n) <robert@carosi.nl>
 Roberto Santalla <roobre@users.noreply.github.com>
 Robin Schoonover <robin@cornhooves.org>
 Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
+Ross Smith II (rasa) <ross@smithii.com>
 rubenbe <github-com-00ff86@vandamme.email>
 Ruslan Yevdokymov <38809160+ruslanye@users.noreply.github.com>
 Ryan Qian <i@bitbili.net>
@@ -274,29 +300,29 @@ Scott Klupfel (kluppy) <kluppy@going2blue.com>
 sec65 <106604020+sec65@users.noreply.github.com>
 Sergey Mishin (ralder) <ralder@yandex.ru>
 Sertonix <83883937+Sertonix@users.noreply.github.com>
-Severin von Wnuck-Lipinski <ss7@live.de>
 Shaarad Dalvi <60266155+shaaraddalvi@users.noreply.github.com> <shdalv@microsoft.com>
+Simon Frei (imsodin) <freisim93@gmail.com>
 Simon Mwepu <simonmwepu@gmail.com>
-Simon Pickup <simon@pickupinfinity.com>
 Sly_tom_cat <slytomcat@mail.ru>
-Sonu Kumar Saw <31889738+dev-saw99@users.noreply.github.com>
 Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
+Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
 Steven Eckhoff <steven.eckhoff.opensource@gmail.com>
 Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
 Sven Bachmann <dev@mcbachmann.de>
-Sébastien WENSKE <sebastien@wenske.fr>
+Syncthing Automation <automation@syncthing.net>
+Syncthing Release Automation <release@syncthing.net>
 Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
-Terrance <git@terrance.allofti.me>
-TheCreeper <TheCreeper@users.noreply.github.com>
 Thomas <9749173+uhthomas@users.noreply.github.com>
 Thomas Hipp <thomashipp@gmail.com>
 Tim Abell (timabell) <tim@timwise.co.uk>
 Tim Howes (timhowes) <timhowes@berkeley.edu>
-Tobias Frölich <40638719+tobifroe@users.noreply.github.com>
+Tim Nordenfur <tim@gurka.se>
 Tobias Klauser <tobias.klauser@gmail.com>
 Tobias Nygren (tnn2) <tnn@nygren.pp.se>
 Tobias Tom (tobiastom) <t.tom@succont.de>
 Tom Jakubowski <tom@crystae.net>
+Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com> <twilczynski@naver.com>
+Tommy Thorn <tommy-github-email@thorn.ws>
 Tully Robinson (tojrobinson) <tully@tojr.org>
 Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
 Tyler Kropp <kropptyler@gmail.com>
@@ -309,14 +335,13 @@ Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
 villekalliomaki <53118179+villekalliomaki@users.noreply.github.com>
 Vladimir Rusinov <vrusinov@google.com> <vladimir.rusinov@gmail.com>
 wangguoliang <liangcszzu@163.com>
-WangXi <xib1102@icloud.com>
 Will Rouesnel <wrouesnel@wrouesnel.com>
 William A. Kennington III (wkennington) <william@wkennington.com>
 wouter bolsterlee <wouter@bolsterl.ee>
+Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
 xarx00 <xarx00@users.noreply.github.com>
 Xavier O. (damajor) <damajor@gmail.com>
-xjtdy888 (xjtdy888) <xjtdy888@163.com> <xjtdy888@gmail.com>
+xjtdy888 (xjtdy888) <xjtdy888@163.com>
 Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
-yparitcher <y@paritcher.com>
 佛跳墙 <daoquan@qq.com>
 落心 <luoxin.ttt@gmail.com>

CONTRIBUTING.md

@@ -34,163 +34,19 @@ Note that the previously used service at
 retired and we kindly ask you to sign up on Weblate for continued
 involvement.
 
+## Contributing Code
+
+Every contribution is welcome. If you want to contribute but are unsure
+where to start, any open issues are fair game! See the [Contribution
+Guidelines](https://docs.syncthing.net/dev/contributing.html) for the full
+story on committing code.
+
 ## Contributing Documentation
 
 Updates to the [documentation site](https://docs.syncthing.net/) can be
 made as pull requests on the [documentation
 repository](https://github.com/syncthing/docs).
 
-## Contributing Code
-
-Every contribution is welcome. If you want to contribute but are unsure
-where to start, any open issues are fair game! Here's a short rundown of
-what you need to keep in mind:
-
-- Don't worry. You are not expected to get everything right on the first
-  attempt, we'll guide you through it.
-
-- Make sure there is an
-  [issue](https://github.com/syncthing/syncthing/issues) that describes the
-  change you want to do. If the thing you want to do does not have an issue
-  yet, please file one before starting work on it.
-
-- Fork the repository and make your changes in a new branch. Once it's ready
-  for review, create a pull request.
-
-### Authorship
-
-All code authors are listed in the AUTHORS file. When your first pull
-request is accepted your details are added to the AUTHORS file and the list
-of authors in the GUI. Commits must be made with the same name and email as
-listed in the AUTHORS file. To accomplish this, ensure that your git
-configuration is set correctly prior to making your first commit:
-
-    $ git config --global user.name "Jane Doe"
-    $ git config --global user.email janedoe@example.com
-
-You must be reachable on the given email address. If you do not wish to use
-your real name for whatever reason, using a nickname or pseudonym is
-perfectly acceptable.
-
-### The Developer Certificate of Origin (DCO)
-
-The Syncthing project requires the Developer Certificate of Origin (DCO)
-sign-off on pull requests (PRs). This means that all commit messages must
-contain a signature line to indicate that the developer accepts the DCO.
-
-The DCO is a lightweight way for contributors to certify that they wrote (or
-otherwise have the right to submit) the code and changes they are
-contributing to the project. Here is the full [text of the
-DCO](https://developercertificate.org):
-
----
-
-By making a contribution to this project, I certify that:
-
-1. The contribution was created in whole or in part by me and I have the
-   right to submit it under the open source license indicated in the file;
-   or
-
-2. The contribution is based upon previous work that, to the best of my
-   knowledge, is covered under an appropriate open source license and I have
-   the right under that license to submit that work with modifications,
-   whether created in whole or in part by me, under the same open source
-   license (unless I am permitted to submit under a different license), as
-   indicated in the file; or
-
-3. The contribution was provided directly to me by some other person who
-   certified (1), (2) or (3) and I have not modified it.
-
-4. I understand and agree that this project and the contribution are public
-   and that a record of the contribution (including all personal information
-   I submit with it, including my sign-off) is maintained indefinitely and
-   may be redistributed consistent with this project or the open source
-   license(s) involved.
-
----
-
-Contributors indicate that they adhere to these requirements by adding
-a `Signed-off-by` line to their commit messages.  For example:
-
-    This is my commit message
-
-    Signed-off-by: Random J Developer <random@developer.example.org>
-
-The name and email address in this line must match those of the committing
-author, and be the same as what you want in the AUTHORS file as per above.
-
-### Coding Style
-
-#### General
-
-- All text files use Unix line endings. The git settings already present in
-  the repository attempt to enforce this.
-
-- When making changes, follow the brace and parenthesis style of the
-  surrounding code.
-
-#### Go Specific
-
-- Follow the conventions laid out in [Effective
-  Go](https://go.dev/doc/effective_go) as much as makes sense. The review
-  guidelines in [Go Code Review
-  Comments](https://github.com/golang/go/wiki/CodeReviewComments) should
-  generally be followed.
-
-- Each commit should be `go fmt` clean.
-
-- Imports are grouped per `goimports` standard; that is, standard
-  library first, then third party libraries after a blank line.
-
-### Commits
-
-- Commit messages (and pull request titles) should follow the [conventional
-  commits](https://www.conventionalcommits.org/en/v1.0.0/) specification and
-  be in lower case.
-
-- We use a scope description in the commit message subject. This is the
-  component of Syncthing that the commit affects. For example, `gui`,
-  `protocol`, `scanner`, `upnp`, etc -- typically, the part after
-  `internal/`, `lib/` or `cmd/` in the package path. If the commit doesn't
-  affect a specific component, such as for changes to the build system or
-  documentation, the scope should be omitted. The same goes for changes that
-  affect many components which would be cumbersome to list.
-
-- Commits that resolve an existing issue must include the issue number
-  as `(fixes #123)` at the end of the commit message subject. A correctly
-  formatted commit message subject looks like this:
-
-      feat(dialer): add env var to disable proxy fallback (fixes #3006)
-
-- If the commit message subject doesn't say it all, one or more paragraphs of
-  describing text should be added to the commit message. This should explain
-  why the change is made and what it accomplishes.
-
-- When drafting a pull request, please feel free to add commits with
-  corrections and merge from `main` when necessary. This provides a clear time
-  line with changes and simplifies review. Do not, in general, rebase your
-  commits, as this makes review harder.
-
-- Pull requests are merged to `main` using squash merge. The "stream of
-  consciousness" set of commits described in the previous point will be reduced
-  to a single commit at merge time. The pull request title and description will
-  be used as the commit message.
-
-### Tests
-
-Yes please, do add tests when adding features or fixing bugs. Also, when a
-pull request is filed a number of automatic tests are run on the code. This
-includes:
-
-- That the code actually builds and the test suite passes.
-
-- That the code is correctly formatted (`go fmt`).
-
-- That the commits are based on a reasonably recent `main`.
-
-- That the output from `go lint` and `go vet` is clean. (This checks for a
-  number of potential problems the compiler doesn't catch.)
-
 ## Licensing
 
 All contributions are made available under the same license as the already
@@ -203,6 +59,10 @@ otherwise stated this means MPLv2, but there are exceptions:
 - The documentation (man/...) is licensed under the Creative Commons
   Attribution 4.0 International License.
 
+- Projects under vendor/... are copyright by and licensed from their
+  respective original authors. Contributions should be made to the original
+  project, not here.
+
 Regardless of the license in effect, you retain the copyright to your
 contribution.
 

Dockerfile.strelaypoolsrv

@@ -11,6 +11,14 @@ LABEL org.opencontainers.image.authors="The Syncthing Project" \
 
 EXPOSE 8080
 
-COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv
+RUN apk add --no-cache ca-certificates su-exec curl
+ENV PUID=1000 PGID=1000 MAXMIND_KEY=
 
-ENTRYPOINT ["/bin/strelaypoolsrv", "-listen", ":8080"]
+RUN mkdir /var/strelaypoolsrv && chown 1000 /var/strelaypoolsrv
+USER 1000
+
+COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv
+COPY script/strelaypoolsrv-entrypoint.sh /bin/entrypoint.sh
+
+WORKDIR /var/strelaypoolsrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaypoolsrv", "-listen", ":8080"]

README-Docker.md

@@ -23,7 +23,47 @@ example `UMASK=002`.
 **Docker cli**
 ```
 $ docker pull syncthing/syncthing
-$ docker run --network=host  -e STGUIADDRESS= \
+$ docker run -p 8384:8384 -p 22000:22000/tcp -p 22000:22000/udp -p 21027:21027/udp \
+    -v /wherever/st-sync:/var/syncthing \
+    --hostname=my-syncthing \
+    syncthing/syncthing:latest
+```
+
+**Docker compose**
+```yml
+---
+version: "3"
+services:
+  syncthing:
+    image: syncthing/syncthing
+    container_name: syncthing
+    hostname: my-syncthing
+    environment:
+      - PUID=1000
+      - PGID=1000
+    volumes:
+      - /wherever/st-sync:/var/syncthing
+    ports:
+      - 8384:8384 # Web UI
+      - 22000:22000/tcp # TCP file transfers
+      - 22000:22000/udp # QUIC file transfers
+      - 21027:21027/udp # Receive local discovery broadcasts
+    restart: unless-stopped
+```
+
+## Discovery
+
+Note that Docker's default network mode prevents local IP addresses from
+being discovered, as Syncthing is only able to see the internal IP of the
+container on the `172.17.0.0/16` subnet. This will result in poor transfer rates
+if local device addresses are not manually configured.
+
+It is therefore advisable to use the [host network mode](https://docs.docker.com/network/host/) instead:
+
+**Docker cli**
+```
+$ docker pull syncthing/syncthing
+$ docker run --network=host \
     -v /wherever/st-sync:/var/syncthing \
     syncthing/syncthing:latest
 ```
@@ -40,39 +80,33 @@ services:
     environment:
       - PUID=1000
       - PGID=1000
-      - STGUIADDRESS=
     volumes:
       - /wherever/st-sync:/var/syncthing
     network_mode: host
     restart: unless-stopped
-    healthcheck:
-      test: curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -o --color=never OK || exit 1
-      interval: 1m
-      timeout: 10s
-      retries: 3
 ```
 
-## Discovery
-
-Please note that Docker's default network mode prevents local IP addresses
-from being discovered, as Syncthing can only see the internal IP address of
-the container on the `172.17.0.0/16` subnet. This would likely break the ability
-for nodes to establish LAN connections properly, resulting in poor transfer
-rates unless local device addresses are configured manually.
-
-It is therefore strongly recommended to stick to the [host network mode](https://docs.docker.com/network/host/),
-as shown above.
-
 Be aware that syncthing alone is now in control of what interfaces and ports it
 listens on. You can edit the syncthing configuration to change the defaults if
 there are conflicts.
 
 ## GUI Security
 
-By default Syncthing inside the Docker image listens on `0.0.0.0:8384`. This
-allows GUI connections when running without host network mode. The example
-above unsets the `STGUIADDRESS` environment variable to have Syncthing fall
-back to listening on what has been configured in the configuration file or the
-GUI settings dialog. By default this is the localhost IP address `127.0.0.1`.
-If you configure your GUI to be externally reachable, make sure you set up
-authentication and enable TLS.
+By default Syncthing inside the Docker image listens on 0.0.0.0:8384 to
+allow GUI connections via the Docker proxy. This is set by the
+`STGUIADDRESS` environment variable in the Dockerfile, as it differs from
+what Syncthing would otherwise use by default. This means you should set up
+authentication in the GUI, like for any other externally reachable Syncthing
+instance. If you do not require the GUI, or you use host networking, you can
+unset the `STGUIADDRESS` variable to have Syncthing fall back to listening
+on 127.0.0.1:
+
+```
+$ docker pull syncthing/syncthing
+$ docker run -e STGUIADDRESS= \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+With the environment variable unset Syncthing will follow what is set in the
+configuration file / GUI settings dialog.

README.md

@@ -82,11 +82,13 @@ build process.
 
 ## Signed Releases
 
-Release binaries are GPG signed with the key available from
-https://syncthing.net/security/. There is also a built-in automatic
-upgrade mechanism (disabled in some distribution channels) which uses a
-compiled in ECDSA signature. macOS and Windows binaries are also
-code-signed.
+As of v0.10.15 and onwards, release binaries are GPG signed with the key
+D26E6ED000654A3E, available from https://syncthing.net/security/ and
+most key servers.
+
+There is also a built-in automatic upgrade mechanism (disabled in some
+distribution channels) which uses a compiled in ECDSA signature. macOS
+binaries are also properly code signed.
 
 ## Documentation
 

buf.gen.yaml

@@ -1,12 +0,0 @@
-version: v2
-managed:
-  enabled: true
-  override:
-    - file_option: go_package_prefix
-      value: github.com/syncthing/syncthing/internal/gen
-plugins:
-  - remote: buf.build/protocolbuffers/go:v1.35.1
-    out: .
-    opt: module=github.com/syncthing/syncthing
-inputs:
-  - directory: proto

buf.yaml

@@ -1,10 +0,0 @@
-version: v2
-modules:
-  - path: proto
-    name: github.com/syncthing/syncthing
-lint:
-  use:
-    - STANDARD
-breaking:
-  use:
-    - WIRE_JSON

build.go

@@ -4,8 +4,8 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//go:build tools
-// +build tools
+//go:build ignore
+// +build ignore
 
 package main
 
@@ -15,6 +15,7 @@ import (
 	"bytes"
 	"compress/flate"
 	"compress/gzip"
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"flag"
@@ -32,32 +33,32 @@ import (
 	"text/template"
 	"time"
 
+	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	buildpkg "github.com/syncthing/syncthing/lib/build"
-	"github.com/syncthing/syncthing/lib/upgrade"
-	"sigs.k8s.io/yaml"
 )
 
 var (
-	goarch        string
-	goos          string
-	noupgrade     bool
-	version       string
-	goCmd         string
-	race          bool
-	debug         = os.Getenv("BUILDDEBUG") != ""
-	extraTags     string
-	installSuffix string
-	pkgdir        string
-	cc            string
-	run           string
-	benchRun      string
-	buildOut      string
-	debugBinary   bool
-	coverage      bool
-	long          bool
-	timeout       = "120s"
-	longTimeout   = "600s"
-	numVersions   = 5
+	goarch         string
+	goos           string
+	noupgrade      bool
+	version        string
+	goCmd          string
+	race           bool
+	debug          = os.Getenv("BUILDDEBUG") != ""
+	extraTags      string
+	installSuffix  string
+	pkgdir         string
+	cc             string
+	run            string
+	benchRun       string
+	buildOut       string
+	debugBinary    bool
+	coverage       bool
+	long           bool
+	timeout        = "120s"
+	longTimeout    = "600s"
+	numVersions    = 5
+	withNextGenGUI = os.Getenv("BUILD_NEXT_GEN_GUI") != ""
 )
 
 type target struct {
@@ -84,6 +85,7 @@ var targets = map[string]target{
 	"all": {
 		// Only valid for the "build" and "install" commands as it lacks all
 		// the archive creation stuff. buildPkgs gets filled out in init()
+		tags: []string{"purego"},
 	},
 	"syncthing": {
 		// The default target for "build", "install", "tar", "zip", "deb", etc.
@@ -94,40 +96,41 @@ var targets = map[string]target{
 		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/syncthing"},
 		binaryName:  "syncthing", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
-			{src: "{{binary}}", dst: "{{binary}}", perm: 0o755},
-			{src: "README.md", dst: "README.txt", perm: 0o644},
-			{src: "LICENSE", dst: "LICENSE.txt", perm: 0o644},
-			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0o644},
+			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
+			{src: "README.md", dst: "README.txt", perm: 0644},
+			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 			// All files from etc/ and extra/ added automatically in init().
 		},
 		systemdService: "syncthing@*.service",
 		installationFiles: []archiveFile{
-			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0o755},
-			{src: "README.md", dst: "deb/usr/share/doc/syncthing/README.txt", perm: 0o644},
-			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing/LICENSE.txt", perm: 0o644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing/AUTHORS.txt", perm: 0o644},
-			{src: "man/syncthing.1", dst: "deb/usr/share/man/man1/syncthing.1", perm: 0o644},
-			{src: "man/syncthing-config.5", dst: "deb/usr/share/man/man5/syncthing-config.5", perm: 0o644},
-			{src: "man/syncthing-stignore.5", dst: "deb/usr/share/man/man5/syncthing-stignore.5", perm: 0o644},
-			{src: "man/syncthing-device-ids.7", dst: "deb/usr/share/man/man7/syncthing-device-ids.7", perm: 0o644},
-			{src: "man/syncthing-event-api.7", dst: "deb/usr/share/man/man7/syncthing-event-api.7", perm: 0o644},
-			{src: "man/syncthing-faq.7", dst: "deb/usr/share/man/man7/syncthing-faq.7", perm: 0o644},
-			{src: "man/syncthing-networking.7", dst: "deb/usr/share/man/man7/syncthing-networking.7", perm: 0o644},
-			{src: "man/syncthing-rest-api.7", dst: "deb/usr/share/man/man7/syncthing-rest-api.7", perm: 0o644},
-			{src: "man/syncthing-security.7", dst: "deb/usr/share/man/man7/syncthing-security.7", perm: 0o644},
-			{src: "man/syncthing-versioning.7", dst: "deb/usr/share/man/man7/syncthing-versioning.7", perm: 0o644},
-			{src: "etc/linux-systemd/system/syncthing@.service", dst: "deb/lib/systemd/system/syncthing@.service", perm: 0o644},
-			{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0o644},
-			{src: "etc/linux-sysctl/30-syncthing.conf", dst: "deb/usr/lib/sysctl.d/30-syncthing.conf", perm: 0o644},
-			{src: "etc/firewall-ufw/syncthing", dst: "deb/etc/ufw/applications.d/syncthing", perm: 0o644},
-			{src: "etc/linux-desktop/syncthing-start.desktop", dst: "deb/usr/share/applications/syncthing-start.desktop", perm: 0o644},
-			{src: "etc/linux-desktop/syncthing-ui.desktop", dst: "deb/usr/share/applications/syncthing-ui.desktop", perm: 0o644},
-			{src: "assets/logo-32.png", dst: "deb/usr/share/icons/hicolor/32x32/apps/syncthing.png", perm: 0o644},
-			{src: "assets/logo-64.png", dst: "deb/usr/share/icons/hicolor/64x64/apps/syncthing.png", perm: 0o644},
-			{src: "assets/logo-128.png", dst: "deb/usr/share/icons/hicolor/128x128/apps/syncthing.png", perm: 0o644},
-			{src: "assets/logo-256.png", dst: "deb/usr/share/icons/hicolor/256x256/apps/syncthing.png", perm: 0o644},
-			{src: "assets/logo-512.png", dst: "deb/usr/share/icons/hicolor/512x512/apps/syncthing.png", perm: 0o644},
-			{src: "assets/logo-only.svg", dst: "deb/usr/share/icons/hicolor/scalable/apps/syncthing.svg", perm: 0o644},
+			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
+			{src: "README.md", dst: "deb/usr/share/doc/syncthing/README.txt", perm: 0644},
+			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing/AUTHORS.txt", perm: 0644},
+			{src: "man/syncthing.1", dst: "deb/usr/share/man/man1/syncthing.1", perm: 0644},
+			{src: "man/syncthing-config.5", dst: "deb/usr/share/man/man5/syncthing-config.5", perm: 0644},
+			{src: "man/syncthing-stignore.5", dst: "deb/usr/share/man/man5/syncthing-stignore.5", perm: 0644},
+			{src: "man/syncthing-device-ids.7", dst: "deb/usr/share/man/man7/syncthing-device-ids.7", perm: 0644},
+			{src: "man/syncthing-event-api.7", dst: "deb/usr/share/man/man7/syncthing-event-api.7", perm: 0644},
+			{src: "man/syncthing-faq.7", dst: "deb/usr/share/man/man7/syncthing-faq.7", perm: 0644},
+			{src: "man/syncthing-networking.7", dst: "deb/usr/share/man/man7/syncthing-networking.7", perm: 0644},
+			{src: "man/syncthing-rest-api.7", dst: "deb/usr/share/man/man7/syncthing-rest-api.7", perm: 0644},
+			{src: "man/syncthing-security.7", dst: "deb/usr/share/man/man7/syncthing-security.7", perm: 0644},
+			{src: "man/syncthing-versioning.7", dst: "deb/usr/share/man/man7/syncthing-versioning.7", perm: 0644},
+			{src: "etc/linux-systemd/system/syncthing@.service", dst: "deb/lib/systemd/system/syncthing@.service", perm: 0644},
+			{src: "etc/linux-systemd/system/syncthing-resume.service", dst: "deb/lib/systemd/system/syncthing-resume.service", perm: 0644},
+			{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0644},
+			{src: "etc/linux-sysctl/30-syncthing.conf", dst: "deb/usr/lib/sysctl.d/30-syncthing.conf", perm: 0644},
+			{src: "etc/firewall-ufw/syncthing", dst: "deb/etc/ufw/applications.d/syncthing", perm: 0644},
+			{src: "etc/linux-desktop/syncthing-start.desktop", dst: "deb/usr/share/applications/syncthing-start.desktop", perm: 0644},
+			{src: "etc/linux-desktop/syncthing-ui.desktop", dst: "deb/usr/share/applications/syncthing-ui.desktop", perm: 0644},
+			{src: "assets/logo-32.png", dst: "deb/usr/share/icons/hicolor/32x32/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-64.png", dst: "deb/usr/share/icons/hicolor/64x64/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-128.png", dst: "deb/usr/share/icons/hicolor/128x128/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-256.png", dst: "deb/usr/share/icons/hicolor/256x256/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-512.png", dst: "deb/usr/share/icons/hicolor/512x512/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-only.svg", dst: "deb/usr/share/icons/hicolor/scalable/apps/syncthing.svg", perm: 0644},
 		},
 	},
 	"stdiscosrv": {
@@ -139,22 +142,23 @@ var targets = map[string]target{
 		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/stdiscosrv"},
 		binaryName:  "stdiscosrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
-			{src: "{{binary}}", dst: "{{binary}}", perm: 0o755},
-			{src: "cmd/stdiscosrv/README.md", dst: "README.txt", perm: 0o644},
-			{src: "LICENSE", dst: "LICENSE.txt", perm: 0o644},
-			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0o644},
+			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
+			{src: "cmd/stdiscosrv/README.md", dst: "README.txt", perm: 0644},
+			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
 		systemdService: "stdiscosrv.service",
 		installationFiles: []archiveFile{
-			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0o755},
-			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0o644},
-			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0o644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0o644},
-			{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0o644},
-			{src: "cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service", dst: "deb/lib/systemd/system/stdiscosrv.service", perm: 0o644},
-			{src: "cmd/stdiscosrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-discosrv", perm: 0o644},
-			{src: "cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv", dst: "deb/etc/ufw/applications.d/stdiscosrv", perm: 0o644},
+			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
+			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0644},
+			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0644},
+			{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service", dst: "deb/lib/systemd/system/stdiscosrv.service", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-discosrv", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv", dst: "deb/etc/ufw/applications.d/stdiscosrv", perm: 0644},
 		},
+		tags: []string{"purego"},
 	},
 	"strelaysrv": {
 		name:        "strelaysrv",
@@ -165,47 +169,61 @@ var targets = map[string]target{
 		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaysrv"},
 		binaryName:  "strelaysrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
-			{src: "{{binary}}", dst: "{{binary}}", perm: 0o755},
-			{src: "cmd/strelaysrv/README.md", dst: "README.txt", perm: 0o644},
-			{src: "cmd/strelaysrv/LICENSE", dst: "LICENSE.txt", perm: 0o644},
-			{src: "LICENSE", dst: "LICENSE.txt", perm: 0o644},
-			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0o644},
+			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
+			{src: "cmd/strelaysrv/README.md", dst: "README.txt", perm: 0644},
+			{src: "cmd/strelaysrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
 		systemdService: "strelaysrv.service",
 		installationFiles: []archiveFile{
-			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0o755},
-			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0o644},
-			{src: "cmd/strelaysrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0o644},
-			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0o644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0o644},
-			{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0o644},
-			{src: "cmd/strelaysrv/etc/linux-systemd/strelaysrv.service", dst: "deb/lib/systemd/system/strelaysrv.service", perm: 0o644},
-			{src: "cmd/strelaysrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-relaysrv", perm: 0o644},
-			{src: "cmd/strelaysrv/etc/firewall-ufw/strelaysrv", dst: "deb/etc/ufw/applications.d/strelaysrv", perm: 0o644},
+			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
+			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0644},
+			{src: "cmd/strelaysrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
+			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0644},
+			{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0644},
+			{src: "cmd/strelaysrv/etc/linux-systemd/strelaysrv.service", dst: "deb/lib/systemd/system/strelaysrv.service", perm: 0644},
+			{src: "cmd/strelaysrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-relaysrv", perm: 0644},
+			{src: "cmd/strelaysrv/etc/firewall-ufw/strelaysrv", dst: "deb/etc/ufw/applications.d/strelaysrv", perm: 0644},
 		},
 	},
 	"strelaypoolsrv": {
 		name:        "strelaypoolsrv",
+		debname:     "syncthing-relaypoolsrv",
+		debdeps:     []string{"libc6"},
 		description: "Syncthing Relay Pool Server",
-		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/infra/strelaypoolsrv"},
-		binaryName:  "strelaypoolsrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaypoolsrv"},
+		binaryName:  "strelaypoolsrv", // .exe will be added automatically for Windows builds
+		archiveFiles: []archiveFile{
+			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
+			{src: "cmd/strelaypoolsrv/README.md", dst: "README.txt", perm: 0644},
+			{src: "cmd/strelaypoolsrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
+		},
+		installationFiles: []archiveFile{
+			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
+			{src: "cmd/strelaypoolsrv/README.md", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/README.txt", perm: 0644},
+			{src: "cmd/strelaypoolsrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/AUTHORS.txt", perm: 0644},
+		},
 	},
 	"stupgrades": {
 		name:        "stupgrades",
 		description: "Syncthing Upgrade Check Server",
-		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/infra/stupgrades"},
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/stupgrades"},
 		binaryName:  "stupgrades",
 	},
 	"stcrashreceiver": {
 		name:        "stcrashreceiver",
 		description: "Syncthing Crash Server",
-		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/infra/stcrashreceiver"},
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/stcrashreceiver"},
 		binaryName:  "stcrashreceiver",
 	},
 	"ursrv": {
 		name:        "ursrv",
 		description: "Syncthing Usage Reporting Server",
-		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/infra/ursrv"},
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/ursrv"},
 		binaryName:  "ursrv",
 	},
 }
@@ -214,11 +232,15 @@ func initTargets() {
 	all := targets["all"]
 	pkgs, _ := filepath.Glob("cmd/*")
 	for _, pkg := range pkgs {
-		if files, err := filepath.Glob(pkg + "/*.go"); err != nil || len(files) == 0 {
-			// No go files in the directory
+		pkg = filepath.Base(pkg)
+		if strings.HasPrefix(pkg, ".") {
+			// ignore dotfiles
 			continue
 		}
-		all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/%s", pkg))
+		if noupgrade && pkg == "stupgrades" {
+			continue
+		}
+		all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/cmd/%s", pkg))
 	}
 	targets["all"] = all
 
@@ -226,13 +248,13 @@ func initTargets() {
 	// and "extra" dirs.
 	syncthingPkg := targets["syncthing"]
 	for _, file := range listFiles("etc") {
-		syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0o644})
+		syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0644})
 	}
 	for _, file := range listFiles("extra") {
-		syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0o644})
+		syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0644})
 	}
 	for _, file := range listFiles("extra") {
-		syncthingPkg.installationFiles = append(syncthingPkg.installationFiles, archiveFile{src: file, dst: "deb/usr/share/doc/syncthing/" + filepath.Base(file), perm: 0o644})
+		syncthingPkg.installationFiles = append(syncthingPkg.installationFiles, archiveFile{src: file, dst: "deb/usr/share/doc/syncthing/" + filepath.Base(file), perm: 0644})
 	}
 	targets["syncthing"] = syncthingPkg
 }
@@ -288,10 +310,10 @@ func runCommand(cmd string, target target) {
 		build(target, tags)
 
 	case "test":
-		test(strings.Fields(extraTags), "github.com/syncthing/syncthing/internal/...", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
+		test(strings.Fields(extraTags), "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
 
 	case "bench":
-		bench(strings.Fields(extraTags), "github.com/syncthing/syncthing/internal/...", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
+		bench(strings.Fields(extraTags), "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
 
 	case "integration":
 		integration(false)
@@ -322,14 +344,12 @@ func runCommand(cmd string, target target) {
 
 	case "tar":
 		buildTar(target, tags)
-		writeCompatJSON()
 
 	case "zip":
 		buildZip(target, tags)
-		writeCompatJSON()
 
 	case "deb":
-		buildDeb(target, tags)
+		buildDeb(target)
 
 	case "vet":
 		metalintShort()
@@ -379,6 +399,7 @@ func parseFlags() {
 	flag.IntVar(&numVersions, "num-versions", numVersions, "Number of versions for changelog command")
 	flag.StringVar(&run, "run", "", "Specify which tests to run")
 	flag.StringVar(&benchRun, "bench", "", "Specify which benchmarks to run")
+	flag.BoolVar(&withNextGenGUI, "with-next-gen-gui", withNextGenGUI, "Also build 'newgui'")
 	flag.StringVar(&buildOut, "build-out", "", "Set the '-o' value for 'go build'")
 	flag.Parse()
 }
@@ -386,6 +407,7 @@ func parseFlags() {
 func test(tags []string, pkgs ...string) {
 	lazyRebuildAssets()
 
+	tags = append(tags, "purego")
 	args := []string{"test", "-tags", strings.Join(tags, " ")}
 	if long {
 		timeout = longTimeout
@@ -419,7 +441,7 @@ func bench(tags []string, pkgs ...string) {
 func integration(bench bool) {
 	lazyRebuildAssets()
 	args := []string{"test", "-v", "-timeout", "60m", "-tags"}
-	tags := "integration"
+	tags := "purego,integration"
 	if bench {
 		tags += ",benchmark"
 	}
@@ -451,6 +473,10 @@ func benchArgs() []string {
 }
 
 func install(target target, tags []string) {
+	if (target.name == "syncthing" || target.name == "") && !withNextGenGUI {
+		log.Println("Notice: Next generation GUI will not be built; see --with-next-gen-gui.")
+	}
+
 	lazyRebuildAssets()
 
 	tags = append(target.tags, tags...)
@@ -474,12 +500,16 @@ func install(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	args := []string{"install"}
+	args := []string{"install", "-v"}
 	args = appendParameters(args, tags, target.buildPkgs...)
 	runPrint(goCmd, args...)
 }
 
 func build(target target, tags []string) {
+	if (target.name == "syncthing" || target.name == "") && !withNextGenGUI {
+		log.Println("Notice: Next generation GUI will not be built; see --with-next-gen-gui.")
+	}
+
 	lazyRebuildAssets()
 	tags = append(target.tags, tags...)
 
@@ -502,7 +532,7 @@ func build(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	args := []string{"build"}
+	args := []string{"build", "-v"}
 	if buildOut != "" {
 		args = append(args, "-o", buildOut)
 	}
@@ -514,6 +544,13 @@ func setBuildEnvVars() {
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
 	os.Setenv("CC", cc)
+	if os.Getenv("CGO_ENABLED") == "" {
+		switch goos {
+		case "darwin", "solaris":
+		default:
+			os.Setenv("CGO_ENABLED", "0")
+		}
+	}
 }
 
 func appendParameters(args []string, tags []string, pkgs ...string) []string {
@@ -592,7 +629,7 @@ func buildZip(target target, tags []string) {
 	fmt.Println(filename)
 }
 
-func buildDeb(target target, tags []string) {
+func buildDeb(target target) {
 	os.RemoveAll("deb")
 
 	// "goarch" here is set to whatever the Debian packages expect. We correct
@@ -606,7 +643,7 @@ func buildDeb(target target, tags []string) {
 		goarch = "arm"
 	}
 
-	build(target, append(tags, "noupgrade"))
+	build(target, []string{"noupgrade"})
 
 	for i := range target.installationFiles {
 		target.installationFiles[i].src = strings.Replace(target.installationFiles[i].src, "{{binary}}", target.BinaryName(), 1)
@@ -628,9 +665,6 @@ func buildDeb(target target, tags []string) {
 		// than just 0.14.26. This rectifies that.
 		debver = strings.Replace(debver, "-", "~", -1)
 	}
-	if strings.Contains(debver, "_") {
-		debver = strings.Replace(debver, "_", "~", -1)
-	}
 	args := []string{
 		"-t", "deb",
 		"-s", "dir",
@@ -718,7 +752,7 @@ func shouldBuildSyso(dir string) (string, error) {
 	}
 
 	jsonPath := filepath.Join(dir, "versioninfo.json")
-	err = os.WriteFile(jsonPath, bs, 0o644)
+	err = os.WriteFile(jsonPath, bs, 0644)
 	if err != nil {
 		return "", errors.New("failed to create " + jsonPath + ": " + err.Error())
 	}
@@ -732,9 +766,12 @@ func shouldBuildSyso(dir string) (string, error) {
 	sysoPath := filepath.Join(dir, "cmd", "syncthing", "resource.syso")
 
 	// See https://github.com/josephspurrier/goversioninfo#command-line-flags
-	arm := strings.HasPrefix(goarch, "arm")
-	a64 := strings.Contains(goarch, "64")
-	if _, err := runError("goversioninfo", "-o", sysoPath, fmt.Sprintf("-arm=%v", arm), fmt.Sprintf("-64=%v", a64)); err != nil {
+	armOption := ""
+	if strings.Contains(goarch, "arm") {
+		armOption = "-arm=true"
+	}
+
+	if _, err := runError("goversioninfo", "-o", sysoPath, armOption); err != nil {
 		return "", errors.New("failed to create " + sysoPath + ": " + err.Error())
 	}
 
@@ -774,7 +811,7 @@ func copyFile(src, dst string, perm os.FileMode) error {
 	}
 
 copy:
-	os.MkdirAll(filepath.Dir(dst), 0o777)
+	os.MkdirAll(filepath.Dir(dst), 0777)
 	if err := os.WriteFile(dst, in, perm); err != nil {
 		return err
 	}
@@ -799,18 +836,50 @@ func listFiles(dir string) []string {
 
 func rebuildAssets() {
 	os.Setenv("SOURCE_DATE_EPOCH", fmt.Sprint(buildStamp()))
-	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/api/auto", "github.com/syncthing/syncthing/cmd/infra/strelaypoolsrv/auto")
+	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/api/auto", "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto")
 }
 
 func lazyRebuildAssets() {
 	shouldRebuild := shouldRebuildAssets("lib/api/auto/gui.files.go", "gui") ||
-		shouldRebuildAssets("cmd/infra/strelaypoolsrv/auto/gui.files.go", "cmd/infra/strelaypoolsrv/gui")
+		shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.files.go", "cmd/strelaypoolsrv/gui")
+
+	if withNextGenGUI {
+		shouldRebuild = buildNextGenGUI() || shouldRebuild
+	}
 
 	if shouldRebuild {
 		rebuildAssets()
 	}
 }
 
+func buildNextGenGUI() bool {
+	// Check if we need to run the npm process, and if so also set the flag
+	// to rebuild Go assets afterwards. The index.html is regenerated every
+	// time by the build process. This assumes the new GUI ends up in
+	// next-gen-gui/dist/next-gen-gui.
+
+	if !shouldRebuildAssets("gui/next-gen-gui/index.html", "next-gen-gui") {
+		// The GUI is up to date.
+		return false
+	}
+
+	runPrintInDir("next-gen-gui", "npm", "install")
+	runPrintInDir("next-gen-gui", "npm", "run", "build", "--", "--prod", "--subresource-integrity")
+
+	rmr("gui/tech-ui")
+
+	for _, src := range listFiles("next-gen-gui/dist") {
+		rel, _ := filepath.Rel("next-gen-gui/dist", src)
+		dst := filepath.Join("gui", rel)
+		if err := copyFile(src, dst, 0644); err != nil {
+			fmt.Println("copy:", err)
+			os.Exit(1)
+		}
+	}
+
+	return true
+}
+
 func shouldRebuildAssets(target, srcdir string) bool {
 	info, err := os.Stat(target)
 	if err != nil {
@@ -858,9 +927,22 @@ func updateDependencies() {
 }
 
 func proto() {
-	// buf needs to be installed
-	// https://buf.build/docs/installation/
-	runPrint("buf", "generate")
+	pv := protobufVersion()
+	repo := "https://github.com/gogo/protobuf.git"
+	path := filepath.Join("repos", "protobuf")
+
+	runPrint(goCmd, "install", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
+	os.MkdirAll("repos", 0755)
+
+	if _, err := os.Stat(path); err != nil {
+		runPrint("git", "clone", repo, path)
+	} else {
+		runPrintInDir(path, "git", "fetch")
+	}
+	runPrintInDir(path, "git", "checkout", pv)
+
+	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/cmd/stdiscosrv")
+	runPrint(goCmd, "generate", "proto/generate.go")
 }
 
 func testmocks() {
@@ -870,6 +952,7 @@ func testmocks() {
 		"github.com/syncthing/syncthing/lib/connections",
 		"github.com/syncthing/syncthing/lib/discover",
 		"github.com/syncthing/syncthing/lib/events",
+		"github.com/syncthing/syncthing/lib/logger",
 		"github.com/syncthing/syncthing/lib/model",
 		"github.com/syncthing/syncthing/lib/protocol",
 	}
@@ -900,7 +983,6 @@ func weblate() {
 func ldflags(tags []string) string {
 	b := new(strings.Builder)
 	b.WriteString("-w")
-	b.WriteString(" -buildid=")
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Version=%s", version)
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp=%d", buildStamp())
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User=%s", buildUser())
@@ -922,9 +1004,6 @@ func rmr(paths ...string) {
 }
 
 func getReleaseVersion() (string, error) {
-	if ver := os.Getenv("VERSION"); ver != "" {
-		return strings.TrimSpace(ver), nil
-	}
 	bs, err := os.ReadFile("RELEASE")
 	if err != nil {
 		return "", err
@@ -969,7 +1048,7 @@ func getGitVersion() (string, error) {
 }
 
 func getVersion() string {
-	// First try for a RELEASE file or $VERSION env var,
+	// First try for a RELEASE file,
 	if ver, err := getReleaseVersion(); err == nil {
 		return ver
 	}
@@ -1298,7 +1377,10 @@ func zipFile(out string, files []archiveFile) {
 }
 
 func codesign(target target) {
-	if goos == "darwin" {
+	switch goos {
+	case "windows":
+		windowsCodesign(target.BinaryName())
+	case "darwin":
 		macosCodesign(target.BinaryName())
 	}
 }
@@ -1322,6 +1404,70 @@ func macosCodesign(file string) {
 	}
 }
 
+func windowsCodesign(file string) {
+	st := "signtool.exe"
+
+	if path := os.Getenv("CODESIGN_SIGNTOOL"); path != "" {
+		st = path
+	}
+
+	for i, algo := range []string{"sha1", "sha256"} {
+		args := []string{"sign", "/fd", algo}
+		if f := os.Getenv("CODESIGN_CERTIFICATE_FILE"); f != "" {
+			args = append(args, "/f", f)
+		} else if b := os.Getenv("CODESIGN_CERTIFICATE_BASE64"); b != "" {
+			// Decode the PFX certificate from base64.
+			bs, err := base64.RawStdEncoding.DecodeString(b)
+			if err != nil {
+				log.Println("Codesign: signing failed: decoding base64:", err)
+				return
+			}
+
+			// Write it to a temporary file
+			f, err := os.CreateTemp("", "codesign-*.pfx")
+			if err != nil {
+				log.Println("Codesign: signing failed: creating temp file:", err)
+				return
+			}
+			_ = f.Chmod(0600) // best effort remove other users' access
+			defer os.Remove(f.Name())
+			if _, err := f.Write(bs); err != nil {
+				log.Println("Codesign: signing failed: writing temp file:", err)
+				return
+			}
+			if err := f.Close(); err != nil {
+				log.Println("Codesign: signing failed: closing temp file:", err)
+				return
+			}
+
+			// Use that when signing
+			args = append(args, "/f", f.Name())
+		}
+		if p := os.Getenv("CODESIGN_CERTIFICATE_PASSWORD"); p != "" {
+			args = append(args, "/p", p)
+		}
+		if tr := os.Getenv("CODESIGN_TIMESTAMP_SERVER"); tr != "" {
+			switch algo {
+			case "sha256":
+				args = append(args, "/tr", tr, "/td", algo)
+			default:
+				args = append(args, "/t", tr)
+			}
+		}
+		if i > 0 {
+			args = append(args, "/as")
+		}
+		args = append(args, file)
+
+		bs, err := runError(st, args...)
+		if err != nil {
+			log.Printf("Codesign: signing failed: %v: %s", err, string(bs))
+			return
+		}
+		log.Println("Codesign: successfully signed", file, "using", algo)
+	}
+}
+
 func metalint() {
 	lazyRebuildAssets()
 	runPrint(goCmd, "test", "-run", "Metalint", "./meta")
@@ -1339,6 +1485,14 @@ func (t target) BinaryName() string {
 	return t.binaryName
 }
 
+func protobufVersion() string {
+	bs, err := runError(goCmd, "list", "-f", "{{.Version}}", "-m", "github.com/gogo/protobuf")
+	if err != nil {
+		log.Fatal("Getting protobuf version:", err)
+	}
+	return string(bs)
+}
+
 func currentAndLatestVersions(n int) ([]string, error) {
 	bs, err := runError("git", "tag", "--sort", "taggerdate")
 	if err != nil {
@@ -1405,29 +1559,3 @@ func nextPatchVersion(ver string) string {
 	digits[len(digits)-1] = strconv.Itoa(n + 1)
 	return strings.Join(digits, ".")
 }
-
-func writeCompatJSON() {
-	bs, err := os.ReadFile("compat.yaml")
-	if err != nil {
-		log.Fatal("Reading compat.yaml:", err)
-	}
-
-	var entries []upgrade.ReleaseCompatibility
-	if err := yaml.Unmarshal(bs, &entries); err != nil {
-		log.Fatal("Parsing compat.yaml:", err)
-	}
-
-	rt := runtime.Version()
-	for _, e := range entries {
-		if !strings.HasPrefix(rt, e.Runtime) {
-			continue
-		}
-		bs, _ := json.MarshalIndent(e, "", "  ")
-		if err := os.WriteFile("compat.json", bs, 0o644); err != nil {
-			log.Fatal("Writing compat.json:", err)
-		}
-		return
-	}
-
-	log.Fatalf("runtime %v not found in compat.yaml", rt)
-}

build.sh

@@ -23,11 +23,10 @@ case "${1:-default}" in
 
 	prerelease)
 		script authors
-		script copyrights
 		build weblate
 		pushd man ; ./refresh.sh ; popd
 		git add -A gui man AUTHORS
-		git commit -m 'chore(gui, man, authors): update docs, translations, and contributors'
+		git commit -m 'gui, man, authors: Update docs, translations, and contributors'
 		;;
 
 	*)

cmd/infra/stupgrades/main.go

@@ -1,375 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"log/slog"
-	"net"
-	"net/http"
-	"os"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/alecthomas/kong"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/syncthing/syncthing/internal/slogutil"
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
-	"github.com/syncthing/syncthing/lib/httpcache"
-	"github.com/syncthing/syncthing/lib/upgrade"
-)
-
-type cli struct {
-	Listen        string        `default:":8080" help:"Listen address"`
-	MetricsListen string        `default:":8082" help:"Listen address for metrics"`
-	URL           string        `short:"u" default:"https://api.github.com/repos/syncthing/syncthing/releases?per_page=25" help:"GitHub releases url"`
-	Forward       []string      `short:"f" help:"Forwarded pages, format: /path->https://example/com/url"`
-	CacheTime     time.Duration `default:"15m" help:"Cache time"`
-}
-
-func main() {
-	var params cli
-	kong.Parse(&params)
-	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
-		Level: slog.LevelInfo,
-	})))
-
-	if err := server(&params); err != nil {
-		fmt.Printf("Error: %v\n", err)
-		os.Exit(1)
-	}
-}
-
-func server(params *cli) error {
-	if params.MetricsListen != "" {
-		mux := http.NewServeMux()
-		mux.Handle("/metrics", promhttp.Handler())
-		metricsListen, err := net.Listen("tcp", params.MetricsListen)
-		if err != nil {
-			return fmt.Errorf("metrics: %w", err)
-		}
-		slog.Info("Metrics listener started", slogutil.Address(params.MetricsListen))
-		go func() {
-			if err := http.Serve(metricsListen, mux); err != nil {
-				slog.Warn("Metrics server returned", slogutil.Error(err))
-			}
-		}()
-	}
-
-	cache := &cachedReleases{url: params.URL}
-	if err := cache.Update(context.Background()); err != nil {
-		return fmt.Errorf("initial cache update: %w", err)
-	} else {
-		slog.Info("Initial cache update done")
-	}
-
-	go func() {
-		for range time.NewTicker(params.CacheTime).C {
-			slog.Info("Refreshing cached releases", slogutil.URI(params.URL))
-			if err := cache.Update(context.Background()); err != nil {
-				slog.Error("Failed to refresh cached releases", slogutil.URI(params.URL), slogutil.Error(err))
-			}
-		}
-	}()
-
-	ghRels := &githubReleases{cache: cache}
-	mux := http.NewServeMux()
-	mux.HandleFunc("/ping", ghRels.servePing)
-	mux.HandleFunc("/meta.json", ghRels.serveReleases)
-
-	for _, fwd := range params.Forward {
-		path, url, ok := strings.Cut(fwd, "->")
-		if !ok {
-			return fmt.Errorf("invalid forward: %q", fwd)
-		}
-		slog.Info("Forwarding", "from", path, "to", url)
-		name := strings.ReplaceAll(path, "/", "_")
-		mux.Handle(path, httpcache.SinglePath(&proxy{name: name, url: url}, params.CacheTime))
-	}
-
-	srv := &http.Server{
-		Addr:         params.Listen,
-		Handler:      mux,
-		ReadTimeout:  5 * time.Second,
-		WriteTimeout: 10 * time.Second,
-	}
-	srv.SetKeepAlivesEnabled(false)
-
-	srvListener, err := net.Listen("tcp", params.Listen)
-	if err != nil {
-		return fmt.Errorf("listen: %w", err)
-	}
-	slog.Info("Main listener started", slogutil.Address(params.Listen))
-
-	return srv.Serve(srvListener)
-}
-
-type githubReleases struct {
-	cache *cachedReleases
-}
-
-func (p *githubReleases) servePing(w http.ResponseWriter, req *http.Request) {
-	rels := p.cache.Releases()
-
-	if len(rels) == 0 {
-		http.Error(w, "No releases available", http.StatusServiceUnavailable)
-		return
-	}
-
-	w.Header().Set("Syncthing-Num-Releases", strconv.Itoa(len(rels)))
-	w.WriteHeader(http.StatusOK)
-}
-
-func (p *githubReleases) serveReleases(w http.ResponseWriter, req *http.Request) {
-	rels := p.cache.Releases()
-
-	ua := req.Header.Get("User-Agent")
-	osv := req.Header.Get("Syncthing-Os-Version")
-	if ua != "" && osv != "" {
-		// We should determine the compatibility of the releases.
-		rels = filterForCompatibility(rels, ua, osv)
-	} else {
-		metricFilterCalls.WithLabelValues("no-ua-or-osversion").Inc()
-	}
-
-	rels = filterForLatest(rels)
-
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	w.Header().Set("Access-Control-Allow-Origin", "*")
-	w.Header().Set("Access-Control-Allow-Methods", "GET")
-	w.Header().Set("Cache-Control", "public, max-age=900")
-	w.Header().Set("Vary", "User-Agent, Syncthing-Os-Version")
-	_ = json.NewEncoder(w).Encode(rels)
-
-	metricUpgradeChecks.Inc()
-}
-
-type proxy struct {
-	name string
-	url  string
-}
-
-func (p *proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	req, err := http.NewRequestWithContext(req.Context(), http.MethodGet, p.url, nil)
-	if err != nil {
-		metricHTTPRequests.WithLabelValues(p.name, "error").Inc()
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		metricHTTPRequests.WithLabelValues(p.name, "error").Inc()
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-	defer resp.Body.Close()
-	metricHTTPRequests.WithLabelValues(p.name, "success").Inc()
-
-	ct := resp.Header.Get("Content-Type")
-	w.Header().Set("Content-Type", ct)
-	if resp.StatusCode == http.StatusOK {
-		w.Header().Set("Cache-Control", "public, max-age=900")
-		w.Header().Set("Access-Control-Allow-Origin", "*")
-		w.Header().Set("Access-Control-Allow-Methods", "GET")
-	}
-	w.WriteHeader(resp.StatusCode)
-	if strings.HasPrefix(ct, "application/json") {
-		// Special JSON handling; clean it up a bit.
-		var v interface{}
-		if err := json.NewDecoder(resp.Body).Decode(&v); err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-		_ = json.NewEncoder(w).Encode(v)
-	} else {
-		_, _ = io.Copy(w, resp.Body)
-	}
-}
-
-// filterForLatest returns the latest stable and prerelease only. If the
-// stable version is newer (comes first in the list) there is no need to go
-// looking for a prerelease at all.
-func filterForLatest(rels []upgrade.Release) []upgrade.Release {
-	var filtered []upgrade.Release
-	havePre := make(map[string]bool)
-	haveStable := make(map[string]bool)
-	for _, rel := range rels {
-		major, _, _ := strings.Cut(rel.Tag, ".")
-		if !rel.Prerelease && !haveStable[major] {
-			// Remember the first non-pre for each major
-			filtered = append(filtered, rel)
-			haveStable[major] = true
-			continue
-		}
-		if rel.Prerelease && !havePre[major] && !haveStable[major] {
-			// We remember the first prerelease we find, unless we've
-			// already found a non-pre of the same major.
-			filtered = append(filtered, rel)
-			havePre[major] = true
-		}
-	}
-	return filtered
-}
-
-var userAgentOSArchExp = regexp.MustCompile(`^syncthing.*\(.+ (\w+)-(\w+)\)$`)
-
-func filterForCompatibility(rels []upgrade.Release, ua, osv string) []upgrade.Release {
-	osArch := userAgentOSArchExp.FindStringSubmatch(ua)
-	if len(osArch) != 3 {
-		metricFilterCalls.WithLabelValues("bad-os-arch").Inc()
-		return rels
-	}
-	os := osArch[1]
-
-	var filtered []upgrade.Release
-	for _, rel := range rels {
-		if rel.Compatibility == nil {
-			// No requirements means it's compatible with everything.
-			filtered = append(filtered, rel)
-			continue
-		}
-
-		req, ok := rel.Compatibility.Requirements[os]
-		if !ok {
-			// No entry for the current OS means it's compatible.
-			filtered = append(filtered, rel)
-			continue
-		}
-
-		if upgrade.CompareVersions(osv, req) >= 0 {
-			filtered = append(filtered, rel)
-			continue
-		}
-	}
-
-	if len(filtered) != len(rels) {
-		metricFilterCalls.WithLabelValues("filtered").Inc()
-	} else {
-		metricFilterCalls.WithLabelValues("unchanged").Inc()
-	}
-
-	return filtered
-}
-
-type cachedReleases struct {
-	url                  string
-	mut                  sync.RWMutex
-	current              []upgrade.Release
-	latestRel, latestPre string
-}
-
-func (c *cachedReleases) Releases() []upgrade.Release {
-	c.mut.RLock()
-	defer c.mut.RUnlock()
-	return c.current
-}
-
-func (c *cachedReleases) Update(ctx context.Context) error {
-	rels, err := fetchGithubReleases(ctx, c.url)
-	if err != nil {
-		return err
-	}
-	latestRel, latestPre := "", ""
-	for _, rel := range rels {
-		if !rel.Prerelease && latestRel == "" {
-			latestRel = rel.Tag
-		}
-		if rel.Prerelease && latestPre == "" {
-			latestPre = rel.Tag
-		}
-		if latestRel != "" && latestPre != "" {
-			break
-		}
-	}
-	c.mut.Lock()
-	c.current = rels
-	if latestRel != c.latestRel || latestPre != c.latestPre {
-		metricLatestReleaseInfo.DeleteLabelValues(c.latestRel, c.latestPre)
-		metricLatestReleaseInfo.WithLabelValues(latestRel, latestPre).Set(1)
-		c.latestRel = latestRel
-		c.latestPre = latestPre
-	}
-	c.mut.Unlock()
-	return nil
-}
-
-func fetchGithubReleases(ctx context.Context, url string) ([]upgrade.Release, error) {
-	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, nil)
-	if err != nil {
-		metricHTTPRequests.WithLabelValues("github-releases", "error").Inc()
-		return nil, err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		metricHTTPRequests.WithLabelValues("github-releases", "error").Inc()
-		return nil, err
-	}
-	defer resp.Body.Close()
-	var rels []upgrade.Release
-	if err := json.NewDecoder(resp.Body).Decode(&rels); err != nil {
-		metricHTTPRequests.WithLabelValues("github-releases", "error").Inc()
-		return nil, err
-	}
-	metricHTTPRequests.WithLabelValues("github-releases", "success").Inc()
-
-	// Move the URL used for browser downloads to the URL field, and remove
-	// the browser URL field. This avoids going via the GitHub API for
-	// downloads, since Syncthing uses the URL field.
-	for _, rel := range rels {
-		for j, asset := range rel.Assets {
-			rel.Assets[j].URL = asset.BrowserURL
-			rel.Assets[j].BrowserURL = ""
-		}
-	}
-
-	addReleaseCompatibility(ctx, rels)
-
-	sort.Sort(upgrade.SortByRelease(rels))
-	return rels, nil
-}
-
-func addReleaseCompatibility(ctx context.Context, rels []upgrade.Release) {
-	for i := range rels {
-		rel := &rels[i]
-		for i, asset := range rel.Assets {
-			if asset.Name != "compat.json" {
-				continue
-			}
-
-			// Load compat.json into the Compatibility field
-			req, err := http.NewRequestWithContext(ctx, http.MethodGet, asset.URL, nil)
-			if err != nil {
-				metricHTTPRequests.WithLabelValues("compat-json", "error").Inc()
-				break
-			}
-			resp, err := http.DefaultClient.Do(req)
-			if err != nil {
-				metricHTTPRequests.WithLabelValues("compat-json", "error").Inc()
-				break
-			}
-			if resp.StatusCode != http.StatusOK {
-				metricHTTPRequests.WithLabelValues("compat-json", "error").Inc()
-				resp.Body.Close()
-				break
-			}
-			_ = json.NewDecoder(io.LimitReader(resp.Body, 10<<10)).Decode(&rel.Compatibility)
-			metricHTTPRequests.WithLabelValues("compat-json", "success").Inc()
-			resp.Body.Close()
-
-			// Remove compat.json from the asset list since it's been processed
-			rel.Assets = append(rel.Assets[:i], rel.Assets[i+1:]...)
-			break
-		}
-	}
-}

cmd/infra/stupgrades/metrics.go

@@ -1,36 +0,0 @@
-// Copyright (C) 2024 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-var (
-	metricUpgradeChecks = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "upgrade",
-		Name:      "metadata_requests",
-	})
-	metricFilterCalls = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "upgrade",
-		Name:      "filter_calls",
-	}, []string{"result"})
-	metricHTTPRequests = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "upgrade",
-		Name:      "http_requests",
-	}, []string{"target", "result"})
-	metricLatestReleaseInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "syncthing",
-		Subsystem: "upgrade",
-		Name:      "latest_release_info",
-		Help:      "Release information",
-	}, []string{"latest_release", "latest_pre"})
-)

cmd/infra/ursrv/serve/metrics.go

@@ -1,61 +0,0 @@
-// Copyright (C) 2023 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package serve
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-var (
-	metricReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "ursrv_v2",
-		Name:      "incoming_reports_total",
-	}, []string{"result"})
-	metricsCollectsTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "ursrv_v2",
-		Name:      "collects_total",
-	})
-	metricsCollectSecondsTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "ursrv_v2",
-		Name:      "collect_seconds_total",
-	})
-	metricsCollectSecondsLast = promauto.NewGauge(prometheus.GaugeOpts{
-		Namespace: "syncthing",
-		Subsystem: "ursrv_v2",
-		Name:      "collect_seconds_last",
-	})
-	metricsRecalcsTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "ursrv_v2",
-		Name:      "recalcs_total",
-	})
-	metricsRecalcSecondsTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "ursrv_v2",
-		Name:      "recalc_seconds_total",
-	})
-	metricsRecalcSecondsLast = promauto.NewGauge(prometheus.GaugeOpts{
-		Namespace: "syncthing",
-		Subsystem: "ursrv_v2",
-		Name:      "recalc_seconds_last",
-	})
-	metricsWriteSecondsLast = promauto.NewGauge(prometheus.GaugeOpts{
-		Namespace: "syncthing",
-		Subsystem: "ursrv_v2",
-		Name:      "write_seconds_last",
-	})
-)
-
-func init() {
-	metricReportsTotal.WithLabelValues("fail")
-	metricReportsTotal.WithLabelValues("replace")
-	metricReportsTotal.WithLabelValues("accept")
-}

cmd/infra/ursrv/serve/prometheus.go

@@ -1,351 +0,0 @@
-// Copyright (C) 2024 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package serve
-
-import (
-	"context"
-	"fmt"
-	"log/slog"
-	"reflect"
-	"slices"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/syncthing/syncthing/lib/ur/contract"
-)
-
-const namePrefix = "syncthing_usage_"
-
-type metricsSet struct {
-	srv *server
-
-	gauges         map[string]prometheus.Gauge
-	gaugeVecs      map[string]*prometheus.GaugeVec
-	gaugeVecLabels map[string][]string
-	summaries      map[string]*metricSummary
-
-	collectMut    sync.RWMutex
-	collectCutoff time.Duration
-}
-
-func newMetricsSet(srv *server) *metricsSet {
-	s := &metricsSet{
-		srv:            srv,
-		gauges:         make(map[string]prometheus.Gauge),
-		gaugeVecs:      make(map[string]*prometheus.GaugeVec),
-		gaugeVecLabels: make(map[string][]string),
-		summaries:      make(map[string]*metricSummary),
-		collectCutoff:  -24 * time.Hour,
-	}
-
-	var initForType func(reflect.Type)
-	initForType = func(t reflect.Type) {
-		for i := range t.NumField() {
-			field := t.Field(i)
-			if field.Type.Kind() == reflect.Struct {
-				initForType(field.Type)
-				continue
-			}
-			name, typ, label := fieldNameTypeLabel(field)
-			sname, labels := nameConstLabels(name)
-			switch typ {
-			case "gauge":
-				s.gauges[name] = prometheus.NewGauge(prometheus.GaugeOpts{
-					Name:        namePrefix + sname,
-					ConstLabels: labels,
-				})
-			case "summary":
-				s.summaries[name] = newMetricSummary(namePrefix+sname, nil, labels)
-			case "gaugeVec":
-				s.gaugeVecLabels[name] = append(s.gaugeVecLabels[name], label)
-			case "summaryVec":
-				s.summaries[name] = newMetricSummary(namePrefix+sname, []string{label}, labels)
-			}
-		}
-	}
-	initForType(reflect.ValueOf(contract.Report{}).Type())
-
-	for name, labels := range s.gaugeVecLabels {
-		s.gaugeVecs[name] = prometheus.NewGaugeVec(prometheus.GaugeOpts{
-			Name: namePrefix + name,
-		}, labels)
-	}
-
-	return s
-}
-
-func fieldNameTypeLabel(rf reflect.StructField) (string, string, string) {
-	metric := rf.Tag.Get("metric")
-	name, typ, ok := strings.Cut(metric, ",")
-	if !ok {
-		return "", "", ""
-	}
-	gv, label, ok := strings.Cut(typ, ":")
-	if ok {
-		typ = gv
-	}
-	return name, typ, label
-}
-
-func nameConstLabels(name string) (string, prometheus.Labels) {
-	if name == "-" {
-		return "", nil
-	}
-	name, labels, ok := strings.Cut(name, "{")
-	if !ok {
-		return name, nil
-	}
-	lls := strings.Split(labels[:len(labels)-1], ",")
-	m := make(map[string]string)
-	for _, l := range lls {
-		k, v, _ := strings.Cut(l, "=")
-		m[k] = v
-	}
-	return name, m
-}
-
-func (s *metricsSet) Serve(ctx context.Context) error {
-	s.recalc()
-
-	const recalcInterval = 5 * time.Minute
-	next := time.Until(time.Now().Truncate(recalcInterval).Add(recalcInterval))
-	recalcTimer := time.NewTimer(next)
-	defer recalcTimer.Stop()
-
-	for {
-		select {
-		case <-recalcTimer.C:
-			s.recalc()
-			next := time.Until(time.Now().Truncate(recalcInterval).Add(recalcInterval))
-			recalcTimer.Reset(next)
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-}
-
-func (s *metricsSet) recalc() {
-	s.collectMut.Lock()
-	defer s.collectMut.Unlock()
-
-	t0 := time.Now()
-	defer func() {
-		dur := time.Since(t0)
-		slog.Info("Metrics recalculated", "d", dur.String())
-		metricsRecalcSecondsLast.Set(dur.Seconds())
-		metricsRecalcSecondsTotal.Add(dur.Seconds())
-		metricsRecalcsTotal.Inc()
-	}()
-
-	for _, g := range s.gauges {
-		g.Set(0)
-	}
-	for _, g := range s.gaugeVecs {
-		g.Reset()
-	}
-	for _, g := range s.summaries {
-		g.Reset()
-	}
-
-	cutoff := time.Now().Add(s.collectCutoff)
-	s.srv.reports.Range(func(key string, r *contract.Report) bool {
-		if s.collectCutoff < 0 && r.Received.Before(cutoff) {
-			s.srv.reports.Delete(key)
-			return true
-		}
-		s.addReport(r)
-		return true
-	})
-}
-
-func (s *metricsSet) addReport(r *contract.Report) {
-	gaugeVecs := make(map[string][]string)
-	s.addReportStruct(reflect.ValueOf(r).Elem(), gaugeVecs)
-	for name, lv := range gaugeVecs {
-		s.gaugeVecs[name].WithLabelValues(lv...).Add(1)
-	}
-}
-
-func (s *metricsSet) addReportStruct(v reflect.Value, gaugeVecs map[string][]string) {
-	t := v.Type()
-	for i := range v.NumField() {
-		field := v.Field(i)
-		if field.Kind() == reflect.Struct {
-			s.addReportStruct(field, gaugeVecs)
-			continue
-		}
-
-		name, typ, label := fieldNameTypeLabel(t.Field(i))
-		switch typ {
-		case "gauge":
-			switch v := field.Interface().(type) {
-			case int:
-				s.gauges[name].Add(float64(v))
-			case string:
-				s.gaugeVecs[name].WithLabelValues(v).Add(1)
-			case bool:
-				if v {
-					s.gauges[name].Add(1)
-				}
-			}
-		case "gaugeVec":
-			var labelValue string
-			switch v := field.Interface().(type) {
-			case string:
-				labelValue = v
-			case int:
-				labelValue = strconv.Itoa(v)
-			case map[string]int:
-				for k, v := range v {
-					labelValue = k
-					field.SetInt(int64(v))
-					break
-				}
-			}
-			if _, ok := gaugeVecs[name]; !ok {
-				gaugeVecs[name] = make([]string, len(s.gaugeVecLabels[name]))
-			}
-			for i, l := range s.gaugeVecLabels[name] {
-				if l == label {
-					gaugeVecs[name][i] = labelValue
-					break
-				}
-			}
-		case "summary", "summaryVec":
-			switch v := field.Interface().(type) {
-			case int:
-				s.summaries[name].Observe("", float64(v))
-			case float64:
-				s.summaries[name].Observe("", v)
-			case []int:
-				for _, v := range v {
-					s.summaries[name].Observe("", float64(v))
-				}
-			case map[string]int:
-				for k, v := range v {
-					if k == "" {
-						// avoid empty string labels as those are the sign
-						// of a non-vec summary
-						k = "unknown"
-					}
-					s.summaries[name].Observe(k, float64(v))
-				}
-			}
-		}
-	}
-}
-
-func (s *metricsSet) Describe(c chan<- *prometheus.Desc) {
-	for _, g := range s.gauges {
-		g.Describe(c)
-	}
-	for _, g := range s.gaugeVecs {
-		g.Describe(c)
-	}
-	for _, g := range s.summaries {
-		g.Describe(c)
-	}
-}
-
-func (s *metricsSet) Collect(c chan<- prometheus.Metric) {
-	s.collectMut.RLock()
-	defer s.collectMut.RUnlock()
-
-	t0 := time.Now()
-	defer func() {
-		dur := time.Since(t0).Seconds()
-		metricsCollectSecondsLast.Set(dur)
-		metricsCollectSecondsTotal.Add(dur)
-		metricsCollectsTotal.Inc()
-	}()
-
-	for _, g := range s.gauges {
-		c <- g
-	}
-	for _, g := range s.gaugeVecs {
-		g.Collect(c)
-	}
-	for _, g := range s.summaries {
-		g.Collect(c)
-	}
-}
-
-type metricSummary struct {
-	name   string
-	values map[string][]float64
-	zeroes map[string]int
-
-	qDesc     *prometheus.Desc
-	countDesc *prometheus.Desc
-	sumDesc   *prometheus.Desc
-	zDesc     *prometheus.Desc
-}
-
-func newMetricSummary(name string, labels []string, constLabels prometheus.Labels) *metricSummary {
-	return &metricSummary{
-		name:      name,
-		values:    make(map[string][]float64),
-		zeroes:    make(map[string]int),
-		qDesc:     prometheus.NewDesc(name, "", append(labels, "quantile"), constLabels),
-		countDesc: prometheus.NewDesc(name+"_nonzero_count", "", labels, constLabels),
-		sumDesc:   prometheus.NewDesc(name+"_sum", "", labels, constLabels),
-		zDesc:     prometheus.NewDesc(name+"_zero_count", "", labels, constLabels),
-	}
-}
-
-func (q *metricSummary) Observe(labelValue string, v float64) {
-	if v == 0 {
-		q.zeroes[labelValue]++
-		return
-	}
-	q.values[labelValue] = append(q.values[labelValue], v)
-}
-
-func (q *metricSummary) Describe(c chan<- *prometheus.Desc) {
-	c <- q.qDesc
-	c <- q.countDesc
-	c <- q.sumDesc
-	c <- q.zDesc
-}
-
-func (q *metricSummary) Collect(c chan<- prometheus.Metric) {
-	for lv, vs := range q.values {
-		var labelVals []string
-		if lv != "" {
-			labelVals = []string{lv}
-		}
-
-		c <- prometheus.MustNewConstMetric(q.countDesc, prometheus.GaugeValue, float64(len(vs)), labelVals...)
-		c <- prometheus.MustNewConstMetric(q.zDesc, prometheus.GaugeValue, float64(q.zeroes[lv]), labelVals...)
-
-		var sum float64
-		for _, v := range vs {
-			sum += v
-		}
-		c <- prometheus.MustNewConstMetric(q.sumDesc, prometheus.GaugeValue, sum, labelVals...)
-
-		if len(vs) == 0 {
-			return
-		}
-
-		slices.Sort(vs)
-
-		pctiles := []float64{0, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.975, 0.99, 1}
-		for _, pct := range pctiles {
-			idx := int(float64(len(vs)-1) * pct)
-			c <- prometheus.MustNewConstMetric(q.qDesc, prometheus.GaugeValue, vs[idx], append(labelVals, fmt.Sprint(pct))...)
-		}
-	}
-}
-
-func (q *metricSummary) Reset() {
-	clear(q.values)
-	clear(q.zeroes)
-}

cmd/infra/ursrv/serve/serve.go

@@ -1,444 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package serve
-
-import (
-	"bufio"
-	"compress/gzip"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"log/slog"
-	"net"
-	"net/http"
-	_ "net/http/pprof"
-	"os"
-	"regexp"
-	"strings"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/puzpuzpuz/xsync/v3"
-	"github.com/syncthing/syncthing/internal/blob"
-	"github.com/syncthing/syncthing/internal/blob/azureblob"
-	"github.com/syncthing/syncthing/internal/blob/s3"
-	"github.com/syncthing/syncthing/internal/slogutil"
-	"github.com/syncthing/syncthing/lib/build"
-	"github.com/syncthing/syncthing/lib/geoip"
-	"github.com/syncthing/syncthing/lib/ur/contract"
-	"github.com/thejerf/suture/v4"
-)
-
-type CLI struct {
-	Listen          string        `env:"UR_LISTEN" help:"Usage reporting & metrics endpoint listen address" default:"0.0.0.0:8080"`
-	ListenInternal  string        `env:"UR_LISTEN_INTERNAL" help:"Internal metrics endpoint listen address" default:"0.0.0.0:8082"`
-	GeoIPLicenseKey string        `env:"UR_GEOIP_LICENSE_KEY"`
-	GeoIPAccountID  int           `env:"UR_GEOIP_ACCOUNT_ID"`
-	DumpFile        string        `env:"UR_DUMP_FILE" default:"reports.jsons.gz"`
-	DumpInterval    time.Duration `env:"UR_DUMP_INTERVAL" default:"5m"`
-
-	S3Endpoint    string `name:"s3-endpoint" env:"UR_S3_ENDPOINT"`
-	S3Region      string `name:"s3-region" env:"UR_S3_REGION"`
-	S3Bucket      string `name:"s3-bucket" env:"UR_S3_BUCKET"`
-	S3AccessKeyID string `name:"s3-access-key-id" env:"UR_S3_ACCESS_KEY_ID"`
-	S3SecretKey   string `name:"s3-secret-key" env:"UR_S3_SECRET_KEY"`
-
-	AzureBlobAccount   string `name:"azure-blob-account" env:"UR_AZUREBLOB_ACCOUNT"`
-	AzureBlobKey       string `name:"azure-blob-key" env:"UR_AZUREBLOB_KEY"`
-	AzureBlobContainer string `name:"azure-blob-container" env:"UR_AZUREBLOB_CONTAINER"`
-}
-
-var (
-	compilerRe         = regexp.MustCompile(`\(([A-Za-z0-9()., -]+) \w+-\w+(?:| android| default)\) ([\w@.-]+)`)
-	knownDistributions = []distributionMatch{
-		// Maps well known builders to the official distribution method that
-		// they represent
-
-		{regexp.MustCompile(`\steamcity@build\.syncthing\.net`), "GitHub"},
-		{regexp.MustCompile(`\sjenkins@build\.syncthing\.net`), "GitHub"},
-		{regexp.MustCompile(`\sbuilder@github\.syncthing\.net`), "GitHub"},
-
-		{regexp.MustCompile(`\sdeb@build\.syncthing\.net`), "APT"},
-		{regexp.MustCompile(`\sdebian@github\.syncthing\.net`), "APT"},
-
-		{regexp.MustCompile(`\sdocker@syncthing\.net`), "Docker Hub"},
-		{regexp.MustCompile(`\sdocker@build.syncthing\.net`), "Docker Hub"},
-		{regexp.MustCompile(`\sdocker@github.syncthing\.net`), "Docker Hub"},
-
-		{regexp.MustCompile(`\sandroid-builder@github\.syncthing\.net`), "Google Play"},
-		{regexp.MustCompile(`\sandroid-.*teamcity@build\.syncthing\.net`), "Google Play"},
-
-		{regexp.MustCompile(`\sandroid-.*vagrant@basebox-stretch64`), "F-Droid"},
-		{regexp.MustCompile(`\svagrant@bullseye`), "F-Droid"},
-		{regexp.MustCompile(`\svagrant@bookworm`), "F-Droid"},
-
-		{regexp.MustCompile(`\sreproducible-build@Catfriend1-syncthing-android`), "Syncthing-Fork Catfriend1 (3rd party)"},
-		{regexp.MustCompile(`\sreproducible-build@nel0x-syncthing-android-gplay`), "Syncthing-Fork nel0x (3rd party)"},
-
-		{regexp.MustCompile(`\sbuilduser@(archlinux|svetlemodry)`), "Arch (3rd party)"},
-		{regexp.MustCompile(`\ssyncthing@archlinux`), "Arch (3rd party)"},
-		{regexp.MustCompile(`@debian`), "Debian (3rd party)"},
-		{regexp.MustCompile(`@fedora`), "Fedora (3rd party)"},
-		{regexp.MustCompile(`@openSUSE`), "openSUSE (3rd party)"},
-		{regexp.MustCompile(`\sbrew@`), "Homebrew (3rd party)"},
-		{regexp.MustCompile(`\sroot@buildkitsandbox`), "LinuxServer.io (3rd party)"},
-		{regexp.MustCompile(`\sports@freebsd`), "FreeBSD (3rd party)"},
-		{regexp.MustCompile(`\snix@nix`), "Nix (3rd party)"},
-		{regexp.MustCompile(`.`), "Others"},
-	}
-)
-
-type distributionMatch struct {
-	matcher      *regexp.Regexp
-	distribution string
-}
-
-func (cli *CLI) Run() error {
-	slog.Info("Starting", "version", build.Version)
-
-	// Listening
-
-	urListener, err := net.Listen("tcp", cli.Listen)
-	if err != nil {
-		slog.Error("Failed to listen (usage reports)", slogutil.Error(err))
-		return err
-	}
-	slog.Info("Listening (usage reports)", slogutil.Address(urListener.Addr()))
-
-	internalListener, err := net.Listen("tcp", cli.ListenInternal)
-	if err != nil {
-		slog.Error("Failed to listen (internal)", slogutil.Error(err))
-		return err
-	}
-	slog.Info("Listening (internal)", slogutil.Address(internalListener.Addr()))
-
-	var geo *geoip.Provider
-	if cli.GeoIPAccountID != 0 && cli.GeoIPLicenseKey != "" {
-		geo, err = geoip.NewGeoLite2CityProvider(context.Background(), cli.GeoIPAccountID, cli.GeoIPLicenseKey, os.TempDir())
-		if err != nil {
-			slog.Error("Failed to load GeoIP", slogutil.Error(err))
-			return err
-		}
-		go geo.Serve(context.TODO())
-	}
-
-	// Blob storage
-
-	var blobs blob.Store
-	if cli.S3Endpoint != "" {
-		blobs, err = s3.NewSession(cli.S3Endpoint, cli.S3Region, cli.S3Bucket, cli.S3AccessKeyID, cli.S3SecretKey)
-		if err != nil {
-			slog.Error("Failed to create S3 session", slogutil.Error(err))
-			return err
-		}
-	} else if cli.AzureBlobAccount != "" {
-		blobs, err = azureblob.NewBlobStore(cli.AzureBlobAccount, cli.AzureBlobKey, cli.AzureBlobContainer)
-		if err != nil {
-			slog.Error("Failed to create Azure blob store", slogutil.Error(err))
-			return err
-		}
-	}
-
-	if _, err := os.Stat(cli.DumpFile); err != nil && blobs != nil {
-		if err := cli.downloadDumpFile(blobs); err != nil {
-			slog.Error("Failed to download dump file", slogutil.Error(err))
-		}
-	}
-
-	// server
-
-	srv := &server{
-		geo:     geo,
-		reports: xsync.NewMapOf[string, *contract.Report](),
-	}
-
-	if fd, err := os.Open(cli.DumpFile); err == nil {
-		gr, err := gzip.NewReader(fd)
-		if err == nil {
-			srv.load(gr)
-		}
-		fd.Close()
-	}
-
-	go func() {
-		for range time.Tick(cli.DumpInterval) {
-			if err := cli.saveDumpFile(srv, blobs); err != nil {
-				slog.Error("Failed to write dump file", slogutil.Error(err))
-			}
-		}
-	}()
-
-	// The internal metrics endpoint just serves metrics about what the
-	// server is doing.
-
-	http.Handle("/metrics", promhttp.Handler())
-
-	internalSrv := http.Server{
-		ReadTimeout:  5 * time.Second,
-		WriteTimeout: 15 * time.Second,
-	}
-	go internalSrv.Serve(internalListener)
-
-	// New external metrics endpoint accepts reports from clients and serves
-	// aggregated usage reporting metrics.
-
-	main := suture.NewSimple("main")
-	main.ServeBackground(context.Background())
-
-	ms := newMetricsSet(srv)
-	main.Add(ms)
-
-	reg := prometheus.NewRegistry()
-	reg.MustRegister(ms)
-
-	mux := http.NewServeMux()
-	mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
-	mux.HandleFunc("/newdata", srv.handleNewData)
-	mux.HandleFunc("/ping", srv.handlePing)
-
-	metricsSrv := http.Server{
-		ReadTimeout:  5 * time.Second,
-		WriteTimeout: 60 * time.Second,
-		Handler:      mux,
-	}
-
-	slog.Info("Ready to serve")
-	return metricsSrv.Serve(urListener)
-}
-
-func (cli *CLI) downloadDumpFile(blobs blob.Store) error {
-	latestKey, err := blobs.LatestKey(context.Background())
-	if err != nil {
-		return fmt.Errorf("list latest S3 key: %w", err)
-	}
-	fd, err := os.Create(cli.DumpFile)
-	if err != nil {
-		return fmt.Errorf("create dump file: %w", err)
-	}
-	if err := blobs.Download(context.Background(), latestKey, fd); err != nil {
-		_ = fd.Close()
-		return fmt.Errorf("download dump file: %w", err)
-	}
-	if err := fd.Close(); err != nil {
-		return fmt.Errorf("close dump file: %w", err)
-	}
-	slog.Info("Dump file downloaded", "key", latestKey)
-	return nil
-}
-
-func (cli *CLI) saveDumpFile(srv *server, blobs blob.Store) error {
-	t0 := time.Now()
-	defer func() {
-		metricsWriteSecondsLast.Set(float64(time.Since(t0)))
-	}()
-
-	fd, err := os.Create(cli.DumpFile + ".tmp")
-	if err != nil {
-		return fmt.Errorf("creating dump file: %w", err)
-	}
-	gw := gzip.NewWriter(fd)
-	if err := srv.save(gw); err != nil {
-		return fmt.Errorf("saving dump file: %w", err)
-	}
-	if err := gw.Close(); err != nil {
-		fd.Close()
-		return fmt.Errorf("closing gzip writer: %w", err)
-	}
-	if err := fd.Close(); err != nil {
-		return fmt.Errorf("closing dump file: %w", err)
-	}
-	if err := os.Rename(cli.DumpFile+".tmp", cli.DumpFile); err != nil {
-		return fmt.Errorf("renaming dump file: %w", err)
-	}
-	slog.Info("Dump file saved", "d", time.Since(t0).String())
-
-	if blobs != nil {
-		t1 := time.Now()
-		key := fmt.Sprintf("reports-%s.jsons.gz", time.Now().UTC().Format("2006-01-02"))
-		fd, err := os.Open(cli.DumpFile)
-		if err != nil {
-			return fmt.Errorf("opening dump file: %w", err)
-		}
-		if err := blobs.Upload(context.Background(), key, fd); err != nil {
-			return fmt.Errorf("uploading dump file: %w", err)
-		}
-		_ = fd.Close()
-		slog.Info("Dump file uploaded", "d", time.Since(t1).String())
-	}
-
-	return nil
-}
-
-type server struct {
-	geo     *geoip.Provider
-	reports *xsync.MapOf[string, *contract.Report]
-}
-
-func (s *server) handlePing(w http.ResponseWriter, r *http.Request) {
-}
-
-func (s *server) handleNewData(w http.ResponseWriter, r *http.Request) {
-	result := "fail"
-	defer func() {
-		// result is "accept" (new report), "replace" (existing report) or
-		// "fail"
-		metricReportsTotal.WithLabelValues(result).Inc()
-	}()
-
-	defer r.Body.Close()
-
-	if r.Method != http.MethodPost {
-		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
-		return
-	}
-
-	addr := r.Header.Get("X-Forwarded-For")
-	if addr != "" {
-		addr = strings.Split(addr, ", ")[0]
-	} else {
-		addr = r.RemoteAddr
-	}
-
-	if host, _, err := net.SplitHostPort(addr); err == nil {
-		addr = host
-	}
-
-	log := slog.With("addr", addr)
-
-	if net.ParseIP(addr) == nil {
-		addr = ""
-	}
-
-	var rep contract.Report
-
-	lr := &io.LimitedReader{R: r.Body, N: 40 * 1024}
-	bs, _ := io.ReadAll(lr)
-	if err := json.Unmarshal(bs, &rep); err != nil {
-		log.Error("Failed to decode JSON", slogutil.Error(err))
-		http.Error(w, "JSON Decode Error", http.StatusInternalServerError)
-		return
-	}
-
-	rep.Received = time.Now()
-	rep.Date = rep.Received.UTC().Format("20060102")
-	rep.Address = addr
-
-	if err := rep.Validate(); err != nil {
-		log.Error("Failed to validate report", slogutil.Error(err))
-		http.Error(w, "Validation Error", http.StatusInternalServerError)
-		return
-	}
-
-	if s.addReport(&rep) {
-		result = "replace"
-	} else {
-		result = "accept"
-	}
-}
-
-func (s *server) addReport(rep *contract.Report) bool {
-	if s.geo != nil {
-		if ip := net.ParseIP(rep.Address); ip != nil {
-			if city, err := s.geo.City(ip); err == nil {
-				rep.Country = city.Country.Names["en"]
-				rep.CountryCode = city.Country.IsoCode
-			}
-		}
-	}
-	if rep.Country == "" {
-		rep.Country = "Unknown"
-	}
-	if rep.CountryCode == "" {
-		rep.CountryCode = "ZZ"
-	}
-
-	rep.Version = transformVersion(rep.Version)
-	if strings.Contains(rep.Version, ".") {
-		split := strings.SplitN(rep.Version, ".", 3)
-		if len(split) == 3 {
-			rep.MajorVersion = strings.Join(split[:2], ".")
-		}
-	}
-	rep.OS, rep.Arch, _ = strings.Cut(rep.Platform, "-")
-
-	if m := compilerRe.FindStringSubmatch(rep.LongVersion); len(m) == 3 {
-		rep.Compiler = m[1]
-		rep.Builder = m[2]
-	}
-	for _, d := range knownDistributions {
-		if d.matcher.MatchString(rep.LongVersion) {
-			rep.Distribution = d.distribution
-			break
-		}
-	}
-	rep.DistDist = rep.Distribution
-	rep.DistOS = rep.OS
-	rep.DistArch = rep.Arch
-
-	if strings.HasPrefix(rep.Version, "v2.") {
-		rep.Database.ModernCSQLite = strings.Contains(rep.LongVersion, "modernc-sqlite")
-		rep.Database.MattnSQLite = !rep.Database.ModernCSQLite
-	} else {
-		rep.Database.LevelDB = true
-	}
-
-	_, loaded := s.reports.LoadAndStore(rep.UniqueID, rep)
-	return loaded
-}
-
-func (s *server) save(w io.Writer) error {
-	bw := bufio.NewWriter(w)
-	enc := json.NewEncoder(bw)
-	var err error
-	s.reports.Range(func(k string, v *contract.Report) bool {
-		err = enc.Encode(v)
-		return err == nil
-	})
-	if err != nil {
-		return err
-	}
-	return bw.Flush()
-}
-
-func (s *server) load(r io.Reader) {
-	t0 := time.Now()
-	dec := json.NewDecoder(r)
-	s.reports.Clear()
-	for {
-		var rep contract.Report
-		if err := dec.Decode(&rep); errors.Is(err, io.EOF) {
-			break
-		} else if err != nil {
-			slog.Error("Failed to load record", slogutil.Error(err))
-			break
-		}
-		s.addReport(&rep)
-	}
-	slog.Info("Loaded reports", "count", s.reports.Size(), "d", time.Since(t0).String())
-}
-
-var (
-	plusRe  = regexp.MustCompile(`(\+.*|[.-]dev\..*)$`)
-	plusStr = "-dev"
-)
-
-// transformVersion returns a version number formatted correctly, with all
-// development versions aggregated into one.
-func transformVersion(v string) string {
-	if v == "unknown-dev" {
-		return v
-	}
-	if !strings.HasPrefix(v, "v") {
-		v = "v" + v
-	}
-	v = plusRe.ReplaceAllString(v, plusStr)
-
-	return v
-}

cmd/stcompdirs/main.go

@@ -7,7 +7,6 @@
 package main
 
 import (
-	"crypto/sha256"
 	"errors"
 	"flag"
 	"fmt"
@@ -17,6 +16,7 @@ import (
 	"path/filepath"
 
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 func main() {

cmd/stcrashreceiver/diskstore.go

@@ -8,7 +8,6 @@ package main
 
 import (
 	"bytes"
-	"cmp"
 	"compress/gzip"
 	"context"
 	"io"
@@ -16,7 +15,7 @@ import (
 	"math"
 	"os"
 	"path/filepath"
-	"slices"
+	"sort"
 	"time"
 )
 
@@ -178,8 +177,8 @@ func (d *diskStore) inventory() error {
 		})
 		return nil
 	})
-	slices.SortFunc(d.currentFiles, func(a, b currentFile) int {
-		return cmp.Compare(a.mtime, b.mtime)
+	sort.Slice(d.currentFiles, func(i, j int) bool {
+		return d.currentFiles[i].mtime < d.currentFiles[j].mtime
 	})
 	var oldest time.Duration
 	if len(d.currentFiles) > 0 {

cmd/stcrashreceiver/main.go

@@ -14,7 +14,6 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -22,29 +21,25 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"regexp"
-	"strings"
 
 	"github.com/alecthomas/kong"
 	raven "github.com/getsentry/raven-go"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
-	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/sha256"
 	"github.com/syncthing/syncthing/lib/ur"
 )
 
 const maxRequestSize = 1 << 20 // 1 MiB
 
 type cli struct {
-	Dir            string `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."`
-	DSN            string `help:"Sentry DSN" env:"SENTRY_DSN"`
-	Listen         string `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"`
-	MaxDiskFiles   int    `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"`
-	MaxDiskSizeMB  int64  `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"`
-	SentryQueue    int    `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"`
-	DiskQueue      int    `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"`
-	MetricsListen  string `help:"HTTP listen address for metrics" default:":8081" env:"METRICS_LISTEN_ADDRESS"`
-	IgnorePatterns string `help:"File containing ignore patterns (regexp)" env:"IGNORE_PATTERNS" type:"existingfile"`
+	Dir           string `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."`
+	DSN           string `help:"Sentry DSN" env:"SENTRY_DSN"`
+	Listen        string `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"`
+	MaxDiskFiles  int    `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"`
+	MaxDiskSizeMB int64  `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"`
+	SentryQueue   int    `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"`
+	DiskQueue     int    `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"`
 }
 
 func main() {
@@ -67,38 +62,19 @@ func main() {
 	}
 	go ss.Serve(context.Background())
 
-	var ip *ignorePatterns
-	if params.IgnorePatterns != "" {
-		var err error
-		ip, err = loadIgnorePatterns(params.IgnorePatterns)
-		if err != nil {
-			log.Fatalf("Failed to load ignore patterns: %v", err)
-		}
-	}
-
 	cr := &crashReceiver{
 		store:  ds,
 		sentry: ss,
-		ignore: ip,
 	}
 
 	mux.Handle("/", cr)
 	mux.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) {
 		w.Write([]byte("OK"))
 	})
-
-	if params.MetricsListen != "" {
-		mmux := http.NewServeMux()
-		mmux.Handle("/metrics", promhttp.Handler())
-		go func() {
-			if err := http.ListenAndServe(params.MetricsListen, mmux); err != nil {
-				log.Fatalln("HTTP serve metrics:", err)
-			}
-		}()
-	}
+	mux.Handle("/metrics", promhttp.Handler())
 
 	if params.DSN != "" {
-		mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports"), ip))
+		mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports")))
 	}
 
 	log.SetOutput(os.Stdout)
@@ -107,7 +83,7 @@ func main() {
 	}
 }
 
-func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http.ResponseWriter, req *http.Request) {
+func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *http.Request) {
 	return func(w http.ResponseWriter, req *http.Request) {
 		result := "failure"
 		defer func() {
@@ -118,19 +94,14 @@ func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http
 		bs, err := io.ReadAll(lr)
 		req.Body.Close()
 		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		if ignore.match(bs) {
-			result = "ignored"
+			http.Error(w, err.Error(), 500)
 			return
 		}
 
 		var reports []ur.FailureReport
 		err = json.Unmarshal(bs, &reports)
 		if err != nil {
-			http.Error(w, err.Error(), http.StatusBadRequest)
+			http.Error(w, err.Error(), 400)
 			return
 		}
 		if len(reports) == 0 {
@@ -139,9 +110,9 @@ func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http
 			return
 		}
 
-		version, err := build.ParseVersion(reports[0].Version)
+		version, err := parseVersion(reports[0].Version)
 		if err != nil {
-			http.Error(w, err.Error(), http.StatusBadRequest)
+			http.Error(w, err.Error(), 400)
 			return
 		}
 		for _, r := range reports {
@@ -187,42 +158,3 @@ func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string,
 	}
 	return reportServer + path, nil
 }
-
-type ignorePatterns struct {
-	patterns []*regexp.Regexp
-}
-
-func loadIgnorePatterns(path string) (*ignorePatterns, error) {
-	bs, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-
-	var patterns []*regexp.Regexp
-	for _, line := range strings.Split(string(bs), "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" {
-			continue
-		}
-		re, err := regexp.Compile(line)
-		if err != nil {
-			return nil, err
-		}
-		patterns = append(patterns, re)
-	}
-
-	log.Printf("Loaded %d ignore patterns", len(patterns))
-	return &ignorePatterns{patterns: patterns}, nil
-}
-
-func (i *ignorePatterns) match(report []byte) bool {
-	if i == nil {
-		return false
-	}
-	for _, re := range i.patterns {
-		if re.Match(report) {
-			return true
-		}
-	}
-	return false
-}

cmd/stcrashreceiver/sentry.go

@@ -18,7 +18,6 @@ import (
 
 	raven "github.com/getsentry/raven-go"
 	"github.com/maruel/panicparse/v2/stack"
-	"github.com/syncthing/syncthing/lib/build"
 )
 
 const reportServer = "https://crash.syncthing.net/report/"
@@ -90,7 +89,7 @@ func sendReport(dsn string, pkt *raven.Packet, userID string) error {
 	}
 
 	// The client sets release and such on the packet before sending, in the
-	// misguided idea that it knows this better than the packet we give
+	// misguided idea that it knows this better than than the packet we give
 	// it. So we copy the values from the packet to the client first...
 	cli.SetRelease(pkt.Release)
 	cli.SetEnvironment(pkt.Environment)
@@ -106,7 +105,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
 		return nil, errors.New("no first line")
 	}
 
-	version, err := build.ParseVersion(string(parts[0]))
+	version, err := parseVersion(string(parts[0]))
 	if err != nil {
 		return nil, err
 	}
@@ -136,7 +135,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
 
 	r := bytes.NewReader(report)
 	ctx, _, err := stack.ScanSnapshot(r, io.Discard, stack.DefaultOpts())
-	if err != nil && !errors.Is(err, io.EOF) {
+	if err != nil && err != io.EOF {
 		return nil, err
 	}
 	if ctx == nil || len(ctx.Goroutines) == 0 {
@@ -144,12 +143,12 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
 	}
 
 	// Lock the source code loader to the version we are processing here.
-	if version.Commit != "" {
+	if version.commit != "" {
 		// We have a commit hash, so we know exactly which source to use
-		loader.LockWithVersion(version.Commit)
-	} else if strings.HasPrefix(version.Tag, "v") {
+		loader.LockWithVersion(version.commit)
+	} else if strings.HasPrefix(version.tag, "v") {
 		// Lets hope the tag is close enough
-		loader.LockWithVersion(version.Tag)
+		loader.LockWithVersion(version.tag)
 	} else {
 		// Last resort
 		loader.LockWithVersion("main")
@@ -216,26 +215,106 @@ func crashReportFingerprint(message string) []string {
 	return []string{"{{ default }}", message}
 }
 
-func packet(version build.VersionParts, reportType string) *raven.Packet {
+// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]
+// or, somewhere along the way the "+" in the version tag disappeared:
+// syncthing v1.23.7-dev.26.gdf7b56ae.dirty-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]
+var (
+	longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
+	gitExtraRE    = regexp.MustCompile(`\.\d+\.g[0-9a-f]+`) // ".1.g6aaae618"
+	gitExtraSepRE = regexp.MustCompile(`[.-]`)              // dot or dash
+)
+
+type version struct {
+	version  string   // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
+	tag      string   // "v1.1.4-rc.1"
+	commit   string   // "6aaae618", blank when absent
+	codename string   // "Erbium Earthworm"
+	runtime  string   // "go1.12.5"
+	goos     string   // "darwin"
+	goarch   string   // "amd64"
+	builder  string   // "jb@kvin.kastelo.net"
+	extra    []string // "foo", "bar"
+}
+
+func (v version) environment() string {
+	if v.commit != "" {
+		return "Development"
+	}
+	if strings.Contains(v.tag, "-rc.") {
+		return "Candidate"
+	}
+	if strings.Contains(v.tag, "-") {
+		return "Beta"
+	}
+	return "Stable"
+}
+
+func parseVersion(line string) (version, error) {
+	m := longVersionRE.FindStringSubmatch(line)
+	if len(m) == 0 {
+		return version{}, errors.New("unintelligeble version string")
+	}
+
+	v := version{
+		version:  m[1],
+		codename: m[2],
+		runtime:  m[3],
+		goos:     m[4],
+		goarch:   m[5],
+		builder:  m[6],
+	}
+
+	// Split the version tag into tag and commit. This is old style
+	// v1.2.3-something.4+11-g12345678 or newer with just dots
+	// v1.2.3-something.4.11.g12345678 or v1.2.3-dev.11.g12345678.
+	parts := []string{v.version}
+	if strings.Contains(v.version, "+") {
+		parts = strings.Split(v.version, "+")
+	} else {
+		idxs := gitExtraRE.FindStringIndex(v.version)
+		if len(idxs) > 0 {
+			parts = []string{v.version[:idxs[0]], v.version[idxs[0]+1:]}
+		}
+	}
+	v.tag = parts[0]
+	if len(parts) > 1 {
+		fields := gitExtraSepRE.Split(parts[1], -1)
+		if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
+			v.commit = fields[1][1:]
+		}
+	}
+
+	if len(m) >= 8 && m[7] != "" {
+		tags := strings.Split(m[7], ",")
+		for i := range tags {
+			tags[i] = strings.TrimSpace(tags[i])
+		}
+		v.extra = tags
+	}
+
+	return v, nil
+}
+
+func packet(version version, reportType string) *raven.Packet {
 	pkt := &raven.Packet{
 		Platform:    "go",
-		Release:     version.Tag,
-		Environment: version.Environment(),
+		Release:     version.tag,
+		Environment: version.environment(),
 		Tags: raven.Tags{
-			raven.Tag{Key: "version", Value: version.Version},
-			raven.Tag{Key: "tag", Value: version.Tag},
-			raven.Tag{Key: "codename", Value: version.Codename},
-			raven.Tag{Key: "runtime", Value: version.Runtime},
-			raven.Tag{Key: "goos", Value: version.GOOS},
-			raven.Tag{Key: "goarch", Value: version.GOARCH},
-			raven.Tag{Key: "builder", Value: version.Builder},
+			raven.Tag{Key: "version", Value: version.version},
+			raven.Tag{Key: "tag", Value: version.tag},
+			raven.Tag{Key: "codename", Value: version.codename},
+			raven.Tag{Key: "runtime", Value: version.runtime},
+			raven.Tag{Key: "goos", Value: version.goos},
+			raven.Tag{Key: "goarch", Value: version.goarch},
+			raven.Tag{Key: "builder", Value: version.builder},
 			raven.Tag{Key: "report_type", Value: reportType},
 		},
 	}
-	if version.Commit != "" {
-		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.Commit})
+	if version.commit != "" {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
 	}
-	for _, tag := range version.Extra {
+	for _, tag := range version.extra {
 		pkt.Tags = append(pkt.Tags, raven.Tag{Key: tag, Value: "1"})
 	}
 	return pkt

cmd/stcrashreceiver/sentry_test.go

@@ -12,6 +12,66 @@ import (
 	"testing"
 )
 
+func TestParseVersion(t *testing.T) {
+	cases := []struct {
+		longVersion string
+		parsed      version
+	}{
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+			},
+		},
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+				extra:    []string{"foo", "bar"},
+			},
+		},
+		{
+			longVersion: `syncthing v1.23.7-dev.26.gdf7b56ae-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]`,
+			parsed: version{
+				version:  "v1.23.7-dev.26.gdf7b56ae-stversionextra",
+				tag:      "v1.23.7-dev",
+				commit:   "df7b56ae",
+				codename: "Fermium Flea",
+				runtime:  "go1.20.5",
+				goos:     "darwin",
+				goarch:   "arm64",
+				builder:  "jb@ok.kastelo.net",
+				extra:    []string{"Some Wrapper", "purego", "stnoupgrade"},
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		v, err := parseVersion(tc.longVersion)
+		if err != nil {
+			t.Errorf("%s\nerror: %v\n", tc.longVersion, err)
+			continue
+		}
+		if fmt.Sprint(v) != fmt.Sprint(tc.parsed) {
+			t.Errorf("%s\nA: %v\nE: %v\n", tc.longVersion, v, tc.parsed)
+		}
+	}
+}
+
 func TestParseReport(t *testing.T) {
 	bs, err := os.ReadFile("_testdata/panic.log")
 	if err != nil {

cmd/stcrashreceiver/sourcecodeloader.go

@@ -71,6 +71,7 @@ func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]b
 
 		url := urlPrefix + l.version + filename[idx:]
 		resp, err := l.client.Get(url)
+
 		if err != nil {
 			fmt.Println("Loading source:", err)
 			return nil, 0

cmd/stcrashreceiver/stcrashreceiver.go

@@ -12,16 +12,11 @@ import (
 	"net/http"
 	"path"
 	"strings"
-	"sync"
 )
 
 type crashReceiver struct {
 	store  *diskStore
 	sentry *sentryService
-	ignore *ignorePatterns
-
-	ignoredMut sync.RWMutex
-	ignored    map[string]struct{}
 }
 
 func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
@@ -69,12 +64,6 @@ func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http
 // serveHead responds to HEAD requests by checking if the named report
 // already exists in the system.
 func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) {
-	r.ignoredMut.RLock()
-	_, ignored := r.ignored[reportID]
-	r.ignoredMut.RUnlock()
-	if ignored {
-		return // found
-	}
 	if !r.store.Exists(reportID) {
 		http.Error(w, "Not found", http.StatusNotFound)
 	}
@@ -87,15 +76,6 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht
 		metricCrashReportsTotal.WithLabelValues(result).Inc()
 	}()
 
-	r.ignoredMut.RLock()
-	_, ignored := r.ignored[reportID]
-	r.ignoredMut.RUnlock()
-	if ignored {
-		result = "ignored_cached"
-		io.Copy(io.Discard, req.Body)
-		return // found
-	}
-
 	// Read at most maxRequestSize of report data.
 	log.Println("Receiving report", reportID)
 	lr := io.LimitReader(req.Body, maxRequestSize)
@@ -106,17 +86,6 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht
 		return
 	}
 
-	if r.ignore.match(bs) {
-		r.ignoredMut.Lock()
-		if r.ignored == nil {
-			r.ignored = make(map[string]struct{})
-		}
-		r.ignored[reportID] = struct{}{}
-		r.ignoredMut.Unlock()
-		result = "ignored"
-		return
-	}
-
 	result = "success"
 
 	// Store the report

cmd/stcrashreceiver/util.go

@@ -9,13 +9,14 @@ package main
 import (
 	"bytes"
 	"compress/gzip"
-	"crypto/sha256"
-	"encoding/hex"
+	"fmt"
 	"net"
 	"net/http"
 	"os"
 	"path/filepath"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 // userIDFor returns a string we can use as the user ID for the purpose of
@@ -23,7 +24,7 @@ import (
 // remote IP, and the current month.
 func userIDFor(req *http.Request) string {
 	addr := req.RemoteAddr
-	if fwd := req.Header.Get("X-Forwarded-For"); fwd != "" {
+	if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
 		addr = fwd
 	}
 	if host, _, err := net.SplitHostPort(addr); err == nil {
@@ -32,7 +33,7 @@ func userIDFor(req *http.Request) string {
 	now := time.Now().Format("200601")
 	salt := "stcrashreporter"
 	hash := sha256.Sum256([]byte(salt + addr + now))
-	return hex.EncodeToString(hash[:8])
+	return fmt.Sprintf("%x", hash[:8])
 }
 
 // 01234567890abcdef... => 01/23
@@ -52,5 +53,5 @@ func compressAndWrite(bs []byte, fullPath string) error {
 	gw.Close()
 
 	// Create an output file with the compressed report
-	return os.WriteFile(fullPath, buf.Bytes(), 0o644)
+	return os.WriteFile(fullPath, buf.Bytes(), 0644)
 }

cmd/stdisco/main.go

@@ -15,9 +15,6 @@ import (
 	"strings"
 	"time"
 
-	"google.golang.org/protobuf/proto"
-
-	"github.com/syncthing/syncthing/internal/gen/discoproto"
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/beacon"
 	"github.com/syncthing/syncthing/lib/discover"
@@ -78,21 +75,20 @@ func recv(bc beacon.Interface) {
 			continue
 		}
 
-		var ann discoproto.Announce
-		proto.Unmarshal(data[4:], &ann)
+		var ann discover.Announce
+		ann.Unmarshal(data[4:])
 
-		id, _ := protocol.DeviceIDFromBytes(ann.Id)
-		if id == myID {
+		if ann.ID == myID {
 			// This is one of our own fake packets, don't print it.
 			continue
 		}
 
 		// Print announcement details for the first packet from a given
 		// device ID and source address, or if -all was given.
-		key := id.String() + src.String()
+		key := ann.ID.String() + src.String()
 		if all || !seen[key] {
 			log.Printf("Announcement from %v\n", src)
-			log.Printf(" %v at %s\n", id, strings.Join(ann.Addresses, ", "))
+			log.Printf(" %v at %s\n", ann.ID, strings.Join(ann.Addresses, ", "))
 			seen[key] = true
 		}
 	}
@@ -100,11 +96,11 @@ func recv(bc beacon.Interface) {
 
 // sends fake discovery announcements once every second
 func send(bc beacon.Interface) {
-	ann := &discoproto.Announce{
-		Id:        myID[:],
+	ann := discover.Announce{
+		ID:        myID,
 		Addresses: []string{"tcp://fake.example.com:12345"},
 	}
-	bs, _ := proto.Marshal(ann)
+	bs, _ := ann.Marshal()
 
 	for {
 		bc.Send(bs)

cmd/stdiscosrv/amqp.go

@@ -1,262 +0,0 @@
-// Copyright (C) 2024 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"log"
-
-	amqp "github.com/rabbitmq/amqp091-go"
-	"github.com/thejerf/suture/v4"
-	"google.golang.org/protobuf/proto"
-
-	"github.com/syncthing/syncthing/internal/gen/discosrv"
-	"github.com/syncthing/syncthing/internal/protoutil"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-type amqpReplicator struct {
-	suture.Service
-
-	broker   string
-	sender   *amqpSender
-	receiver *amqpReceiver
-	outbox   chan *discosrv.ReplicationRecord
-}
-
-func newAMQPReplicator(broker, clientID string, db database) *amqpReplicator {
-	svc := suture.New("amqpReplicator", suture.Spec{PassThroughPanics: true})
-
-	sender := &amqpSender{
-		broker:   broker,
-		clientID: clientID,
-		outbox:   make(chan *discosrv.ReplicationRecord, replicationOutboxSize),
-	}
-	svc.Add(sender)
-
-	receiver := &amqpReceiver{
-		broker:   broker,
-		clientID: clientID,
-		db:       db,
-	}
-	svc.Add(receiver)
-
-	return &amqpReplicator{
-		Service:  svc,
-		broker:   broker,
-		sender:   sender,
-		receiver: receiver,
-		outbox:   make(chan *discosrv.ReplicationRecord, replicationOutboxSize),
-	}
-}
-
-func (s *amqpReplicator) send(key *protocol.DeviceID, ps []*discosrv.DatabaseAddress, seen int64) {
-	s.sender.send(key, ps, seen)
-}
-
-type amqpSender struct {
-	broker   string
-	clientID string
-	outbox   chan *discosrv.ReplicationRecord
-}
-
-func (s *amqpSender) Serve(ctx context.Context) error {
-	conn, ch, err := amqpChannel(s.broker)
-	if err != nil {
-		return err
-	}
-	defer ch.Close()
-	defer conn.Close()
-
-	buf := make([]byte, 1024)
-	for {
-		select {
-		case rec := <-s.outbox:
-			size := proto.Size(rec)
-			if len(buf) < size {
-				buf = make([]byte, size)
-			}
-
-			n, err := protoutil.MarshalTo(buf, rec)
-			if err != nil {
-				replicationSendsTotal.WithLabelValues("error").Inc()
-				return fmt.Errorf("replication marshal: %w", err)
-			}
-
-			err = ch.PublishWithContext(ctx,
-				"discovery", // exchange
-				"",          // routing key
-				false,       // mandatory
-				false,       // immediate
-				amqp.Publishing{
-					ContentType: "application/protobuf",
-					Body:        buf[:n],
-					AppId:       s.clientID,
-				})
-			if err != nil {
-				replicationSendsTotal.WithLabelValues("error").Inc()
-				return fmt.Errorf("replication publish: %w", err)
-			}
-
-			replicationSendsTotal.WithLabelValues("success").Inc()
-
-		case <-ctx.Done():
-			return nil
-		}
-	}
-}
-
-func (s *amqpSender) String() string {
-	return fmt.Sprintf("amqpSender(%q)", s.broker)
-}
-
-func (s *amqpSender) send(key *protocol.DeviceID, ps []*discosrv.DatabaseAddress, seen int64) {
-	item := &discosrv.ReplicationRecord{
-		Key:       key[:],
-		Addresses: ps,
-		Seen:      seen,
-	}
-
-	// The send should never block. The inbox is suitably buffered for at
-	// least a few seconds of stalls, which shouldn't happen in practice.
-	select {
-	case s.outbox <- item:
-	default:
-		replicationSendsTotal.WithLabelValues("drop").Inc()
-	}
-}
-
-type amqpReceiver struct {
-	broker   string
-	clientID string
-	db       database
-}
-
-func (s *amqpReceiver) Serve(ctx context.Context) error {
-	conn, ch, err := amqpChannel(s.broker)
-	if err != nil {
-		return err
-	}
-	defer ch.Close()
-	defer conn.Close()
-
-	msgs, err := amqpConsume(ch)
-	if err != nil {
-		return err
-	}
-
-	for {
-		select {
-		case msg, ok := <-msgs:
-			if !ok {
-				return fmt.Errorf("subscription closed: %w", io.EOF)
-			}
-
-			// ignore messages from ourself
-			if msg.AppId == s.clientID {
-				continue
-			}
-
-			var rec discosrv.ReplicationRecord
-			if err := proto.Unmarshal(msg.Body, &rec); err != nil {
-				replicationRecvsTotal.WithLabelValues("error").Inc()
-				return fmt.Errorf("replication unmarshal: %w", err)
-			}
-			id, err := protocol.DeviceIDFromBytes(rec.Key)
-			if err != nil {
-				id, err = protocol.DeviceIDFromString(string(rec.Key))
-			}
-			if err != nil {
-				log.Println("Replication device ID:", err)
-				replicationRecvsTotal.WithLabelValues("error").Inc()
-				continue
-			}
-
-			if err := s.db.merge(&id, rec.Addresses, rec.Seen); err != nil {
-				return fmt.Errorf("replication database merge: %w", err)
-			}
-
-			replicationRecvsTotal.WithLabelValues("success").Inc()
-
-		case <-ctx.Done():
-			return nil
-		}
-	}
-}
-
-func (s *amqpReceiver) String() string {
-	return fmt.Sprintf("amqpReceiver(%q)", s.broker)
-}
-
-func amqpChannel(dst string) (*amqp.Connection, *amqp.Channel, error) {
-	conn, err := amqp.Dial(dst)
-	if err != nil {
-		return nil, nil, fmt.Errorf("AMQP dial: %w", err)
-	}
-
-	ch, err := conn.Channel()
-	if err != nil {
-		return nil, nil, fmt.Errorf("AMQP channel: %w", err)
-	}
-
-	err = ch.ExchangeDeclare(
-		"discovery", // name
-		"fanout",    // type
-		false,       // durable
-		false,       // auto-deleted
-		false,       // internal
-		false,       // no-wait
-		nil,         // arguments
-	)
-	if err != nil {
-		return nil, nil, fmt.Errorf("AMQP declare exchange: %w", err)
-	}
-
-	return conn, ch, nil
-}
-
-func amqpConsume(ch *amqp.Channel) (<-chan amqp.Delivery, error) {
-	q, err := ch.QueueDeclare(
-		"",    // name
-		false, // durable
-		false, // delete when unused
-		true,  // exclusive
-		false, // no-wait
-		nil,   // arguments
-	)
-	if err != nil {
-		return nil, fmt.Errorf("AMQP declare queue: %w", err)
-	}
-
-	err = ch.QueueBind(
-		q.Name,      // queue name
-		"",          // routing key
-		"discovery", // exchange
-		false,
-		nil,
-	)
-	if err != nil {
-		return nil, fmt.Errorf("AMQP bind queue: %w", err)
-	}
-
-	msgs, err := ch.Consume(
-		q.Name, // queue
-		"",     // consumer
-		true,   // auto-ack
-		false,  // exclusive
-		false,  // no-local
-		false,  // no-wait
-		nil,    // args
-	)
-	if err != nil {
-		return nil, fmt.Errorf("AMQP consume: %w", err)
-	}
-
-	return msgs, nil
-}

cmd/stdiscosrv/apisrv.go

@@ -22,13 +22,12 @@ import (
 	"net"
 	"net/http"
 	"net/url"
-	"slices"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/syncthing/syncthing/internal/gen/discosrv"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/stringutil"
 )
@@ -40,20 +39,15 @@ type announcement struct {
 }
 
 type apiSrv struct {
-	addr           string
-	cert           tls.Certificate
-	db             database
-	listener       net.Listener
-	repl           replicator // optional
-	useHTTP        bool
-	compression    bool
-	gzipWriters    sync.Pool
-	seenTracker    *retryAfterTracker
-	notSeenTracker *retryAfterTracker
-}
+	addr     string
+	cert     tls.Certificate
+	db       database
+	listener net.Listener
+	repl     replicator // optional
+	useHTTP  bool
 
-type replicator interface {
-	send(key *protocol.DeviceID, addrs []*discosrv.DatabaseAddress, seen int64)
+	mapsMut sync.Mutex
+	misses  map[string]int32
 }
 
 type requestID int64
@@ -66,30 +60,18 @@ type contextKey int
 
 const idKey contextKey = iota
 
-func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP, compression bool, desiredNotFoundRate float64) *apiSrv {
+func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP bool) *apiSrv {
 	return &apiSrv{
-		addr:        addr,
-		cert:        cert,
-		db:          db,
-		repl:        repl,
-		useHTTP:     useHTTP,
-		compression: compression,
-		seenTracker: &retryAfterTracker{
-			name:         "seenTracker",
-			bucketStarts: time.Now(),
-			desiredRate:  desiredNotFoundRate / 2,
-			currentDelay: notFoundRetryUnknownMinSeconds,
-		},
-		notSeenTracker: &retryAfterTracker{
-			name:         "notSeenTracker",
-			bucketStarts: time.Now(),
-			desiredRate:  desiredNotFoundRate / 2,
-			currentDelay: notFoundRetryUnknownMaxSeconds / 2,
-		},
+		addr:    addr,
+		cert:    cert,
+		db:      db,
+		repl:    repl,
+		useHTTP: useHTTP,
+		misses:  make(map[string]int32),
 	}
 }
 
-func (s *apiSrv) Serve(ctx context.Context) error {
+func (s *apiSrv) Serve(_ context.Context) error {
 	if s.useHTTP {
 		listener, err := net.Listen("tcp", s.addr)
 		if err != nil {
@@ -120,15 +102,8 @@ func (s *apiSrv) Serve(ctx context.Context) error {
 		ReadTimeout:    httpReadTimeout,
 		WriteTimeout:   httpWriteTimeout,
 		MaxHeaderBytes: httpMaxHeaderBytes,
+		ErrorLog:       log.New(io.Discard, "", 0),
 	}
-	if !debug {
-		srv.ErrorLog = log.New(io.Discard, "", 0)
-	}
-
-	go func() {
-		<-ctx.Done()
-		srv.Shutdown(context.Background())
-	}()
 
 	err := srv.Serve(s.listener)
 	if err != nil {
@@ -198,7 +173,7 @@ func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
 	deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
 	if err != nil {
 		if debug {
-			log.Println(reqID, "bad device param:", err)
+			log.Println(reqID, "bad device param")
 		}
 		lookupRequestsTotal.WithLabelValues("bad_request").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
@@ -206,7 +181,8 @@ func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	rec, err := s.db.get(&deviceID)
+	key := deviceID.String()
+	rec, err := s.db.get(key)
 	if err != nil {
 		// some sort of internal error
 		lookupRequestsTotal.WithLabelValues("internal_error").Inc()
@@ -216,14 +192,28 @@ func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
 	}
 
 	if len(rec.Addresses) == 0 {
-		var afterS int
-		if rec.Seen == 0 {
-			afterS = s.notSeenTracker.retryAfterS()
-			lookupRequestsTotal.WithLabelValues("not_found_ever").Inc()
+		lookupRequestsTotal.WithLabelValues("not_found").Inc()
+
+		s.mapsMut.Lock()
+		misses := s.misses[key]
+		if misses < rec.Misses {
+			misses = rec.Misses + 1
 		} else {
-			afterS = s.seenTracker.retryAfterS()
-			lookupRequestsTotal.WithLabelValues("not_found_recent").Inc()
+			misses++
 		}
+		s.misses[key] = misses
+		s.mapsMut.Unlock()
+
+		if misses%notFoundMissesWriteInterval == 0 {
+			rec.Misses = misses
+			rec.Missed = time.Now().UnixNano()
+			rec.Addresses = nil
+			// rec.Seen retained from get
+			s.db.put(key, rec)
+		}
+
+		afterS := notFoundRetryAfterSeconds(int(misses))
+		retryAfterHistogram.Observe(float64(afterS))
 		w.Header().Set("Retry-After", strconv.Itoa(afterS))
 		http.Error(w, "Not Found", http.StatusNotFound)
 		return
@@ -235,16 +225,10 @@ func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
 	var bw io.Writer = w
 
 	// Use compression if the client asks for it
-	if s.compression && strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
-		gw, ok := s.gzipWriters.Get().(*gzip.Writer)
-		if ok {
-			gw.Reset(w)
-		} else {
-			gw = gzip.NewWriter(w)
-		}
+	if strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
 		w.Header().Set("Content-Encoding", "gzip")
+		gw := gzip.NewWriter(bw)
 		defer gw.Close()
-		defer s.gzipWriters.Put(gw)
 		bw = gw
 	}
 
@@ -283,9 +267,6 @@ func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req
 
 	addresses := fixupAddresses(remoteAddr, ann.Addresses)
 	if len(addresses) == 0 {
-		if debug {
-			log.Println(reqID, "no addresses")
-		}
 		announceRequestsTotal.WithLabelValues("bad_request").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
 		http.Error(w, "Bad Request", http.StatusBadRequest)
@@ -293,9 +274,6 @@ func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req
 	}
 
 	if err := s.handleAnnounce(deviceID, addresses); err != nil {
-		if debug {
-			log.Println(reqID, "handle:", err)
-		}
 		announceRequestsTotal.WithLabelValues("internal_error").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
@@ -306,9 +284,6 @@ func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req
 
 	w.Header().Set("Reannounce-After", reannounceAfterString())
 	w.WriteHeader(http.StatusNoContent)
-	if debug {
-		log.Println(reqID, "announced", deviceID, addresses)
-	}
 }
 
 func (s *apiSrv) Stop() {
@@ -316,31 +291,29 @@ func (s *apiSrv) Stop() {
 }
 
 func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string) error {
+	key := deviceID.String()
 	now := time.Now()
 	expire := now.Add(addressExpiryTime).UnixNano()
 
+	dbAddrs := make([]DatabaseAddress, len(addresses))
+	for i := range addresses {
+		dbAddrs[i].Address = addresses[i]
+		dbAddrs[i].Expires = expire
+	}
+
 	// The address slice must always be sorted for database merges to work
 	// properly.
-	slices.Sort(addresses)
-	addresses = slices.Compact(addresses)
-
-	dbAddrs := make([]*discosrv.DatabaseAddress, len(addresses))
-	for i := range addresses {
-		dbAddrs[i] = &discosrv.DatabaseAddress{
-			Address: addresses[i],
-			Expires: expire,
-		}
-	}
+	sort.Sort(databaseAddressOrder(dbAddrs))
 
 	seen := now.UnixNano()
 	if s.repl != nil {
-		s.repl.send(&deviceID, dbAddrs, seen)
+		s.repl.send(key, dbAddrs, seen)
 	}
-	return s.db.merge(&deviceID, dbAddrs, seen)
+	return s.db.merge(key, dbAddrs, seen)
 }
 
 func handlePing(w http.ResponseWriter, _ *http.Request) {
-	w.WriteHeader(http.StatusNoContent)
+	w.WriteHeader(204)
 }
 
 func certificateBytes(req *http.Request) ([]byte, error) {
@@ -350,7 +323,7 @@ func certificateBytes(req *http.Request) ([]byte, error) {
 
 	var bs []byte
 
-	if hdr := req.Header.Get("X-Ssl-Cert"); hdr != "" {
+	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
 		if strings.Contains(hdr, "%") {
 			// Nginx using $ssl_client_escaped_cert
 			// The certificate is in PEM format with url encoding.
@@ -386,7 +359,7 @@ func certificateBytes(req *http.Request) ([]byte, error) {
 		}
 
 		bs = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: hdr})
-	} else if cert := req.Header.Get("X-Forwarded-Tls-Client-Cert"); cert != "" {
+	} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
 		// Traefik 2 passtlsclientcert
 		//
 		// The certificate is in PEM format, maybe with URL encoding
@@ -394,36 +367,19 @@ func certificateBytes(req *http.Request) ([]byte, error) {
 		// statements. We need to decode, reinstate the newlines every 64
 		// character and add statements for the PEM decoder
 
-		if strings.Contains(cert, "%") {
-			if unesc, err := url.QueryUnescape(cert); err == nil {
-				cert = unesc
+		if strings.Contains(hdr, "%") {
+			if unesc, err := url.QueryUnescape(hdr); err == nil {
+				hdr = unesc
 			}
 		}
 
-		const (
-			header = "-----BEGIN CERTIFICATE-----"
-			footer = "-----END CERTIFICATE-----"
-		)
-
-		var b bytes.Buffer
-		b.Grow(len(header) + 1 + len(cert) + len(cert)/64 + 1 + len(footer) + 1)
-
-		b.WriteString(header)
-		b.WriteByte('\n')
-
-		for i := 0; i < len(cert); i += 64 {
-			end := i + 64
-			if end > len(cert) {
-				end = len(cert)
-			}
-			b.WriteString(cert[i:end])
-			b.WriteByte('\n')
+		for i := 64; i < len(hdr); i += 65 {
+			hdr = hdr[:i] + "\n" + hdr[i:]
 		}
 
-		b.WriteString(footer)
-		b.WriteByte('\n')
-
-		bs = b.Bytes()
+		hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
+		hdr += "\n-----END CERTIFICATE-----\n"
+		bs = []byte(hdr)
 	}
 
 	if bs == nil {
@@ -488,6 +444,7 @@ func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
 				// remote is nil, unable to determine host IP
 				continue
 			}
+
 		}
 
 		// If zero port was specified, use remote port.
@@ -513,7 +470,6 @@ func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
 
 type loggingResponseWriter struct {
 	http.ResponseWriter
-
 	statusCode int
 }
 
@@ -526,7 +482,7 @@ func (lrw *loggingResponseWriter) WriteHeader(code int) {
 	lrw.ResponseWriter.WriteHeader(code)
 }
 
-func addressStrs(dbAddrs []*discosrv.DatabaseAddress) []string {
+func addressStrs(dbAddrs []DatabaseAddress) []string {
 	res := make([]string, len(dbAddrs))
 	for i, a := range dbAddrs {
 		res[i] = a.Address
@@ -538,44 +494,15 @@ func errorRetryAfterString() string {
 	return strconv.Itoa(errorRetryAfterSeconds + rand.Intn(errorRetryFuzzSeconds))
 }
 
+func notFoundRetryAfterSeconds(misses int) int {
+	retryAfterS := notFoundRetryMinSeconds + notFoundRetryIncSeconds*misses
+	if retryAfterS > notFoundRetryMaxSeconds {
+		retryAfterS = notFoundRetryMaxSeconds
+	}
+	retryAfterS += rand.Intn(notFoundRetryFuzzSeconds)
+	return retryAfterS
+}
+
 func reannounceAfterString() string {
 	return strconv.Itoa(reannounceAfterSeconds + rand.Intn(reannounzeFuzzSeconds))
 }
-
-type retryAfterTracker struct {
-	name        string
-	desiredRate float64 // requests per second
-
-	mut          sync.Mutex
-	lastCount    int       // requests in the last bucket
-	curCount     int       // requests in the current bucket
-	bucketStarts time.Time // start of the current bucket
-	currentDelay int       // current delay in seconds
-}
-
-func (t *retryAfterTracker) retryAfterS() int {
-	now := time.Now()
-	t.mut.Lock()
-	if durS := now.Sub(t.bucketStarts).Seconds(); durS > float64(t.currentDelay) {
-		t.bucketStarts = now
-		t.lastCount = t.curCount
-		lastRate := float64(t.lastCount) / durS
-
-		switch {
-		case t.currentDelay > notFoundRetryUnknownMinSeconds &&
-			lastRate < 0.75*t.desiredRate:
-			t.currentDelay = max(8*t.currentDelay/10, notFoundRetryUnknownMinSeconds)
-		case t.currentDelay < notFoundRetryUnknownMaxSeconds &&
-			lastRate > 1.25*t.desiredRate:
-			t.currentDelay = min(3*t.currentDelay/2, notFoundRetryUnknownMaxSeconds)
-		}
-
-		t.curCount = 0
-	}
-	if t.curCount == 0 {
-		retryAfterLevel.WithLabelValues(t.name).Set(float64(t.currentDelay))
-	}
-	t.curCount++
-	t.mut.Unlock()
-	return t.currentDelay + rand.Intn(t.currentDelay/4)
-}

cmd/stdiscosrv/apisrv_test.go

@@ -7,20 +7,9 @@
 package main
 
 import (
-	"context"
-	"crypto/tls"
 	"fmt"
-	"io"
 	"net"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"regexp"
-	"strings"
 	"testing"
-
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/tlsutil"
 )
 
 func TestFixupAddresses(t *testing.T) {
@@ -105,79 +94,3 @@ func addr(host string, port int) *net.TCPAddr {
 		Port: port,
 	}
 }
-
-func BenchmarkAPIRequests(b *testing.B) {
-	db := newInMemoryStore(b.TempDir(), 0, nil)
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go db.Serve(ctx)
-	api := newAPISrv("127.0.0.1:0", tls.Certificate{}, db, nil, true, true, 1000)
-	srv := httptest.NewServer(http.HandlerFunc(api.handler))
-
-	kf := b.TempDir() + "/cert"
-	crt, err := tlsutil.NewCertificate(kf+".crt", kf+".key", "localhost", 7, true)
-	if err != nil {
-		b.Fatal(err)
-	}
-	certBs, err := os.ReadFile(kf + ".crt")
-	if err != nil {
-		b.Fatal(err)
-	}
-	certBs = regexp.MustCompile(`---[^\n]+---\n`).ReplaceAll(certBs, nil)
-	certString := string(strings.ReplaceAll(string(certBs), "\n", " "))
-
-	devID := protocol.NewDeviceID(crt.Certificate[0])
-	devIDString := devID.String()
-
-	b.Run("Announce", func(b *testing.B) {
-		b.ReportAllocs()
-		url := srv.URL + "/v2/?device=" + devIDString
-		for i := 0; i < b.N; i++ {
-			req, _ := http.NewRequest(http.MethodPost, url, strings.NewReader(`{"addresses":["tcp://10.10.10.10:42000"]}`))
-			req.Header.Set("X-Forwarded-Tls-Client-Cert", certString)
-			resp, err := http.DefaultClient.Do(req)
-			if err != nil {
-				b.Fatal(err)
-			}
-			resp.Body.Close()
-			if resp.StatusCode != http.StatusNoContent {
-				b.Fatalf("unexpected status %s", resp.Status)
-			}
-		}
-	})
-
-	b.Run("Lookup", func(b *testing.B) {
-		b.ReportAllocs()
-		url := srv.URL + "/v2/?device=" + devIDString
-		for i := 0; i < b.N; i++ {
-			req, _ := http.NewRequest(http.MethodGet, url, nil)
-			resp, err := http.DefaultClient.Do(req)
-			if err != nil {
-				b.Fatal(err)
-			}
-			io.Copy(io.Discard, resp.Body)
-			resp.Body.Close()
-			if resp.StatusCode != http.StatusOK {
-				b.Fatalf("unexpected status %s", resp.Status)
-			}
-		}
-	})
-
-	b.Run("LookupNoCompression", func(b *testing.B) {
-		b.ReportAllocs()
-		url := srv.URL + "/v2/?device=" + devIDString
-		for i := 0; i < b.N; i++ {
-			req, _ := http.NewRequest(http.MethodGet, url, nil)
-			req.Header.Set("Accept-Encoding", "identity") // disable compression
-			resp, err := http.DefaultClient.Do(req)
-			if err != nil {
-				b.Fatal(err)
-			}
-			io.Copy(io.Discard, resp.Body)
-			resp.Body.Close()
-			if resp.StatusCode != http.StatusOK {
-				b.Fatalf("unexpected status %s", resp.Status)
-			}
-		}
-	})
-}

cmd/stdiscosrv/database.go

@@ -4,31 +4,23 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
+//go:generate go run ../../proto/scripts/protofmt.go database.proto
+//go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
+
 package main
 
 import (
-	"bufio"
-	"cmp"
 	"context"
-	"encoding/binary"
-	"errors"
-	"io"
 	"log"
-	"os"
-	"path"
-	"runtime"
-	"slices"
-	"strings"
+	"net"
+	"net/url"
+	"sort"
 	"time"
 
-	"github.com/puzpuzpuz/xsync/v3"
-	"google.golang.org/protobuf/proto"
-
-	"github.com/syncthing/syncthing/internal/blob"
-	"github.com/syncthing/syncthing/internal/gen/discosrv"
-	"github.com/syncthing/syncthing/internal/protoutil"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/rand"
+	"github.com/syncthing/syncthing/lib/sliceutil"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 type clock interface {
@@ -42,401 +34,380 @@ func (defaultClock) Now() time.Time {
 }
 
 type database interface {
-	put(key *protocol.DeviceID, rec *discosrv.DatabaseRecord) error
-	merge(key *protocol.DeviceID, addrs []*discosrv.DatabaseAddress, seen int64) error
-	get(key *protocol.DeviceID) (*discosrv.DatabaseRecord, error)
+	put(key string, rec DatabaseRecord) error
+	merge(key string, addrs []DatabaseAddress, seen int64) error
+	get(key string) (DatabaseRecord, error)
 }
 
-type inMemoryStore struct {
-	m             *xsync.MapOf[protocol.DeviceID, *discosrv.DatabaseRecord]
-	dir           string
-	flushInterval time.Duration
-	blobs         blob.Store
-	objKey        string
-	clock         clock
+type levelDBStore struct {
+	db         *leveldb.DB
+	inbox      chan func()
+	clock      clock
+	marshalBuf []byte
 }
 
-func newInMemoryStore(dir string, flushInterval time.Duration, blobs blob.Store) *inMemoryStore {
-	hn, err := os.Hostname()
+func newLevelDBStore(dir string) (*levelDBStore, error) {
+	db, err := leveldb.OpenFile(dir, levelDBOptions)
 	if err != nil {
-		hn = rand.String(8)
-	}
-	s := &inMemoryStore{
-		m:             xsync.NewMapOf[protocol.DeviceID, *discosrv.DatabaseRecord](),
-		dir:           dir,
-		flushInterval: flushInterval,
-		blobs:         blobs,
-		objKey:        hn + ".db",
-		clock:         defaultClock{},
-	}
-	nr, err := s.read()
-	if os.IsNotExist(err) && blobs != nil {
-		// Try to read from blob storage
-		latestKey, cerr := blobs.LatestKey(context.Background())
-		if cerr != nil {
-			log.Println("Error finding database from blob storage:", cerr)
-			return s
-		}
-		fd, cerr := os.Create(path.Join(s.dir, "records.db"))
-		if cerr != nil {
-			log.Println("Error creating database file:", cerr)
-			return s
-		}
-		if cerr := blobs.Download(context.Background(), latestKey, fd); cerr != nil {
-			log.Printf("Error downloading database from blob storage: %v", cerr)
-		}
-		_ = fd.Close()
-		nr, err = s.read()
+		return nil, err
 	}
+	return &levelDBStore{
+		db:    db,
+		inbox: make(chan func(), 16),
+		clock: defaultClock{},
+	}, nil
+}
+
+func newMemoryLevelDBStore() (*levelDBStore, error) {
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
 	if err != nil {
-		log.Println("Error reading database:", err)
+		return nil, err
 	}
-	log.Printf("Read %d records from database", nr)
-	s.expireAndCalculateStatistics()
-	return s
+	return &levelDBStore{
+		db:    db,
+		inbox: make(chan func(), 16),
+		clock: defaultClock{},
+	}, nil
 }
 
-func (s *inMemoryStore) put(key *protocol.DeviceID, rec *discosrv.DatabaseRecord) error {
+func (s *levelDBStore) put(key string, rec DatabaseRecord) error {
 	t0 := time.Now()
-	s.m.Store(*key, rec)
-	databaseOperations.WithLabelValues(dbOpPut, dbResSuccess).Inc()
-	databaseOperationSeconds.WithLabelValues(dbOpPut).Observe(time.Since(t0).Seconds())
-	return nil
+	defer func() {
+		databaseOperationSeconds.WithLabelValues(dbOpPut).Observe(time.Since(t0).Seconds())
+	}()
+
+	rc := make(chan error)
+
+	s.inbox <- func() {
+		size := rec.Size()
+		if len(s.marshalBuf) < size {
+			s.marshalBuf = make([]byte, size)
+		}
+		n, _ := rec.MarshalTo(s.marshalBuf)
+		rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
+	}
+
+	err := <-rc
+	if err != nil {
+		databaseOperations.WithLabelValues(dbOpPut, dbResError).Inc()
+	} else {
+		databaseOperations.WithLabelValues(dbOpPut, dbResSuccess).Inc()
+	}
+
+	return err
 }
 
-func (s *inMemoryStore) merge(key *protocol.DeviceID, addrs []*discosrv.DatabaseAddress, seen int64) error {
+func (s *levelDBStore) merge(key string, addrs []DatabaseAddress, seen int64) error {
 	t0 := time.Now()
+	defer func() {
+		databaseOperationSeconds.WithLabelValues(dbOpMerge).Observe(time.Since(t0).Seconds())
+	}()
 
-	newRec := &discosrv.DatabaseRecord{
+	rc := make(chan error)
+	newRec := DatabaseRecord{
 		Addresses: addrs,
 		Seen:      seen,
 	}
 
-	if oldRec, ok := s.m.Load(*key); ok {
-		newRec = merge(oldRec, newRec)
+	s.inbox <- func() {
+		// grab the existing record
+		oldRec, err := s.get(key)
+		if err != nil {
+			// "not found" is not an error from get, so this is serious
+			// stuff only
+			rc <- err
+			return
+		}
+		newRec = merge(newRec, oldRec)
+
+		// We replicate s.put() functionality here ourselves instead of
+		// calling it because we want to serialize our get above together
+		// with the put in the same function.
+		size := newRec.Size()
+		if len(s.marshalBuf) < size {
+			s.marshalBuf = make([]byte, size)
+		}
+		n, _ := newRec.MarshalTo(s.marshalBuf)
+		rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
 	}
-	s.m.Store(*key, newRec)
 
-	databaseOperations.WithLabelValues(dbOpMerge, dbResSuccess).Inc()
-	databaseOperationSeconds.WithLabelValues(dbOpMerge).Observe(time.Since(t0).Seconds())
+	err := <-rc
+	if err != nil {
+		databaseOperations.WithLabelValues(dbOpMerge, dbResError).Inc()
+	} else {
+		databaseOperations.WithLabelValues(dbOpMerge, dbResSuccess).Inc()
+	}
 
-	return nil
+	return err
 }
 
-func (s *inMemoryStore) get(key *protocol.DeviceID) (*discosrv.DatabaseRecord, error) {
+func (s *levelDBStore) get(key string) (DatabaseRecord, error) {
 	t0 := time.Now()
 	defer func() {
 		databaseOperationSeconds.WithLabelValues(dbOpGet).Observe(time.Since(t0).Seconds())
 	}()
 
-	rec, ok := s.m.Load(*key)
-	if !ok {
+	keyBs := []byte(key)
+	val, err := s.db.Get(keyBs, nil)
+	if err == leveldb.ErrNotFound {
 		databaseOperations.WithLabelValues(dbOpGet, dbResNotFound).Inc()
-		return &discosrv.DatabaseRecord{}, nil
+		return DatabaseRecord{}, nil
+	}
+	if err != nil {
+		databaseOperations.WithLabelValues(dbOpGet, dbResError).Inc()
+		return DatabaseRecord{}, err
 	}
 
-	rec.Addresses = expire(rec.Addresses, s.clock.Now())
+	var rec DatabaseRecord
+
+	if err := rec.Unmarshal(val); err != nil {
+		databaseOperations.WithLabelValues(dbOpGet, dbResUnmarshalError).Inc()
+		return DatabaseRecord{}, nil
+	}
+
+	rec.Addresses = expire(rec.Addresses, s.clock.Now().UnixNano())
 	databaseOperations.WithLabelValues(dbOpGet, dbResSuccess).Inc()
 	return rec, nil
 }
 
-func (s *inMemoryStore) Serve(ctx context.Context) error {
-	if s.flushInterval <= 0 {
-		<-ctx.Done()
-		return nil
-	}
-
-	t := time.NewTimer(s.flushInterval)
+func (s *levelDBStore) Serve(ctx context.Context) error {
+	t := time.NewTimer(0)
 	defer t.Stop()
+	defer s.db.Close()
+
+	// Start the statistics serve routine. It will exit with us when
+	// statisticsTrigger is closed.
+	statisticsTrigger := make(chan struct{})
+	statisticsDone := make(chan struct{})
+	go s.statisticsServe(statisticsTrigger, statisticsDone)
 
 loop:
 	for {
 		select {
+		case fn := <-s.inbox:
+			// Run function in serialized order.
+			fn()
+
 		case <-t.C:
-			log.Println("Calculating statistics")
-			s.expireAndCalculateStatistics()
-			log.Println("Flushing database")
-			if err := s.write(); err != nil {
-				log.Println("Error writing database:", err)
-			}
-			log.Println("Finished flushing database")
-			t.Reset(s.flushInterval)
+			// Trigger the statistics routine to do its thing in the
+			// background.
+			statisticsTrigger <- struct{}{}
+
+		case <-statisticsDone:
+			// The statistics routine is done with one iteratation, schedule
+			// the next.
+			t.Reset(databaseStatisticsInterval)
 
 		case <-ctx.Done():
 			// We're done.
+			close(statisticsTrigger)
 			break loop
 		}
 	}
 
-	return s.write()
-}
-
-func (s *inMemoryStore) expireAndCalculateStatistics() {
-	now := s.clock.Now()
-	cutoff24h := now.Add(-24 * time.Hour).UnixNano()
-	cutoff1w := now.Add(-7 * 24 * time.Hour).UnixNano()
-	current, currentIPv4, currentIPv6, currentIPv6GUA, last24h, last1w := 0, 0, 0, 0, 0, 0
-
-	n := 0
-	s.m.Range(func(key protocol.DeviceID, rec *discosrv.DatabaseRecord) bool {
-		if n%1000 == 0 {
-			runtime.Gosched()
-		}
-		n++
-
-		addresses := expire(rec.Addresses, now)
-		if len(addresses) == 0 {
-			rec.Addresses = nil
-			s.m.Store(key, rec)
-		} else if len(addresses) != len(rec.Addresses) {
-			rec.Addresses = addresses
-			s.m.Store(key, rec)
-		}
-
-		switch {
-		case len(rec.Addresses) > 0:
-			current++
-			seenIPv4, seenIPv6, seenIPv6GUA := false, false, false
-			for _, addr := range rec.Addresses {
-				// We do fast and loose matching on strings here instead of
-				// parsing the address and the IP and doing "proper" checks,
-				// to keep things fast and generate less garbage.
-				if strings.Contains(addr.Address, "[") {
-					seenIPv6 = true
-					if strings.Contains(addr.Address, "[2") {
-						seenIPv6GUA = true
-					}
-				} else {
-					seenIPv4 = true
-				}
-				if seenIPv4 && seenIPv6 && seenIPv6GUA {
-					break
-				}
-			}
-			if seenIPv4 {
-				currentIPv4++
-			}
-			if seenIPv6 {
-				currentIPv6++
-			}
-			if seenIPv6GUA {
-				currentIPv6GUA++
-			}
-		case rec.Seen > cutoff24h:
-			last24h++
-		case rec.Seen > cutoff1w:
-			last1w++
-		default:
-			// drop the record if it's older than a week
-			s.m.Delete(key)
-		}
-		return true
-	})
-
-	databaseKeys.WithLabelValues("current").Set(float64(current))
-	databaseKeys.WithLabelValues("currentIPv4").Set(float64(currentIPv4))
-	databaseKeys.WithLabelValues("currentIPv6").Set(float64(currentIPv6))
-	databaseKeys.WithLabelValues("currentIPv6GUA").Set(float64(currentIPv6GUA))
-	databaseKeys.WithLabelValues("last24h").Set(float64(last24h))
-	databaseKeys.WithLabelValues("last1w").Set(float64(last1w))
-	databaseStatisticsSeconds.Set(time.Since(now).Seconds())
-}
-
-func (s *inMemoryStore) write() (err error) {
-	t0 := time.Now()
-	defer func() {
-		if err == nil {
-			databaseWriteSeconds.Set(time.Since(t0).Seconds())
-			databaseLastWritten.Set(float64(t0.Unix()))
-		}
-	}()
-
-	dbf := path.Join(s.dir, "records.db")
-	fd, err := os.Create(dbf + ".tmp")
-	if err != nil {
-		return err
-	}
-	bw := bufio.NewWriter(fd)
-
-	var buf []byte
-	var rangeErr error
-	now := s.clock.Now()
-	cutoff1w := now.Add(-7 * 24 * time.Hour).UnixNano()
-	n := 0
-	s.m.Range(func(key protocol.DeviceID, value *discosrv.DatabaseRecord) bool {
-		if n%1000 == 0 {
-			runtime.Gosched()
-		}
-		n++
-
-		if value.Seen < cutoff1w {
-			// drop the record if it's older than a week
-			return true
-		}
-		rec := &discosrv.ReplicationRecord{
-			Key:       key[:],
-			Addresses: value.Addresses,
-			Seen:      value.Seen,
-		}
-		s := proto.Size(rec)
-		if s+4 > len(buf) {
-			buf = make([]byte, s+4)
-		}
-		n, err := protoutil.MarshalTo(buf[4:], rec)
-		if err != nil {
-			rangeErr = err
-			return false
-		}
-		binary.BigEndian.PutUint32(buf, uint32(n))
-		if _, err := bw.Write(buf[:n+4]); err != nil {
-			rangeErr = err
-			return false
-		}
-		return true
-	})
-	if rangeErr != nil {
-		_ = fd.Close()
-		return rangeErr
-	}
-
-	if err := bw.Flush(); err != nil {
-		_ = fd.Close
-		return err
-	}
-	if err := fd.Close(); err != nil {
-		return err
-	}
-	if err := os.Rename(dbf+".tmp", dbf); err != nil {
-		return err
-	}
-
-	// Upload to blob storage
-	if s.blobs != nil {
-		fd, err = os.Open(dbf)
-		if err != nil {
-			log.Printf("Error uploading database to blob storage: %v", err)
-			return nil
-		}
-		defer fd.Close()
-		if err := s.blobs.Upload(context.Background(), s.objKey, fd); err != nil {
-			log.Printf("Error uploading database to blob storage: %v", err)
-		}
-		log.Println("Finished uploading database")
-	}
+	// Also wait for statisticsServe to return
+	<-statisticsDone
 
 	return nil
 }
 
-func (s *inMemoryStore) read() (int, error) {
-	fd, err := os.Open(path.Join(s.dir, "records.db"))
-	if err != nil {
-		return 0, err
-	}
-	defer fd.Close()
+func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
+	defer close(done)
 
-	br := bufio.NewReader(fd)
-	var buf []byte
-	nr := 0
-	for {
-		var n uint32
-		if err := binary.Read(br, binary.BigEndian, &n); err != nil {
-			if errors.Is(err, io.EOF) {
-				break
+	for range trigger {
+		t0 := time.Now()
+		nowNanos := t0.UnixNano()
+		cutoff24h := t0.Add(-24 * time.Hour).UnixNano()
+		cutoff1w := t0.Add(-7 * 24 * time.Hour).UnixNano()
+		cutoff2Mon := t0.Add(-60 * 24 * time.Hour).UnixNano()
+		current, currentIPv4, currentIPv6, last24h, last1w, inactive, errors := 0, 0, 0, 0, 0, 0, 0
+
+		iter := s.db.NewIterator(&util.Range{}, nil)
+		for iter.Next() {
+			// Attempt to unmarshal the record and count the
+			// failure if there's something wrong with it.
+			var rec DatabaseRecord
+			if err := rec.Unmarshal(iter.Value()); err != nil {
+				errors++
+				continue
+			}
+
+			// If there are addresses that have not expired it's a current
+			// record, otherwise account it based on when it was last seen
+			// (last 24 hours or last week) or finally as inactice.
+			addrs := expire(rec.Addresses, nowNanos)
+			switch {
+			case len(addrs) > 0:
+				current++
+				seenIPv4, seenIPv6 := false, false
+				for _, addr := range addrs {
+					uri, err := url.Parse(addr.Address)
+					if err != nil {
+						continue
+					}
+					host, _, err := net.SplitHostPort(uri.Host)
+					if err != nil {
+						continue
+					}
+					if ip := net.ParseIP(host); ip != nil && ip.To4() != nil {
+						seenIPv4 = true
+					} else if ip != nil {
+						seenIPv6 = true
+					}
+					if seenIPv4 && seenIPv6 {
+						break
+					}
+				}
+				if seenIPv4 {
+					currentIPv4++
+				}
+				if seenIPv6 {
+					currentIPv6++
+				}
+			case rec.Seen > cutoff24h:
+				last24h++
+			case rec.Seen > cutoff1w:
+				last1w++
+			case rec.Seen > cutoff2Mon:
+				inactive++
+			case rec.Missed < cutoff2Mon:
+				// It hasn't been seen lately and we haven't recorded
+				// someone asking for this device in a long time either;
+				// delete the record.
+				if err := s.db.Delete(iter.Key(), nil); err != nil {
+					databaseOperations.WithLabelValues(dbOpDelete, dbResError).Inc()
+				} else {
+					databaseOperations.WithLabelValues(dbOpDelete, dbResSuccess).Inc()
+				}
+			default:
+				inactive++
 			}
-			return nr, err
-		}
-		if int(n) > len(buf) {
-			buf = make([]byte, n)
-		}
-		if _, err := io.ReadFull(br, buf[:n]); err != nil {
-			return nr, err
-		}
-		rec := &discosrv.ReplicationRecord{}
-		if err := proto.Unmarshal(buf[:n], rec); err != nil {
-			return nr, err
-		}
-		key, err := protocol.DeviceIDFromBytes(rec.Key)
-		if err != nil {
-			key, err = protocol.DeviceIDFromString(string(rec.Key))
-		}
-		if err != nil {
-			log.Println("Bad device ID:", err)
-			continue
 		}
 
-		slices.SortFunc(rec.Addresses, Cmp)
-		rec.Addresses = slices.CompactFunc(rec.Addresses, Equal)
-		s.m.Store(key, &discosrv.DatabaseRecord{
-			Addresses: expire(rec.Addresses, s.clock.Now()),
-			Seen:      rec.Seen,
-		})
-		nr++
+		iter.Release()
+
+		databaseKeys.WithLabelValues("current").Set(float64(current))
+		databaseKeys.WithLabelValues("currentIPv4").Set(float64(currentIPv4))
+		databaseKeys.WithLabelValues("currentIPv6").Set(float64(currentIPv6))
+		databaseKeys.WithLabelValues("last24h").Set(float64(last24h))
+		databaseKeys.WithLabelValues("last1w").Set(float64(last1w))
+		databaseKeys.WithLabelValues("inactive").Set(float64(inactive))
+		databaseKeys.WithLabelValues("error").Set(float64(errors))
+		databaseStatisticsSeconds.Set(time.Since(t0).Seconds())
+
+		// Signal that we are done and can be scheduled again.
+		done <- struct{}{}
 	}
-	return nr, nil
 }
 
 // merge returns the merged result of the two database records a and b. The
 // result is the union of the two address sets, with the newer expiry time
-// chosen for any duplicates. The address list in a is overwritten and
-// reused for the result.
-func merge(a, b *discosrv.DatabaseRecord) *discosrv.DatabaseRecord {
+// chosen for any duplicates.
+func merge(a, b DatabaseRecord) DatabaseRecord {
 	// Both lists must be sorted for this to work.
+	if !sort.IsSorted(databaseAddressOrder(a.Addresses)) {
+		log.Println("Warning: bug: addresses not correctly sorted in merge")
+		a.Addresses = sortedAddressCopy(a.Addresses)
+	}
+	if !sort.IsSorted(databaseAddressOrder(b.Addresses)) {
+		// no warning because this is the side we read from disk and it may
+		// legitimately predate correct sorting.
+		b.Addresses = sortedAddressCopy(b.Addresses)
+	}
 
-	a.Seen = max(a.Seen, b.Seen)
+	res := DatabaseRecord{
+		Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
+		Seen:      a.Seen,
+	}
+	if b.Seen > a.Seen {
+		res.Seen = b.Seen
+	}
 
 	aIdx := 0
 	bIdx := 0
-	for aIdx < len(a.Addresses) && bIdx < len(b.Addresses) {
-		switch cmp.Compare(a.Addresses[aIdx].Address, b.Addresses[bIdx].Address) {
-		case 0:
-			// a == b, choose the newer expiry time
-			a.Addresses[aIdx].Expires = max(a.Addresses[aIdx].Expires, b.Addresses[bIdx].Expires)
+	aAddrs := a.Addresses
+	bAddrs := b.Addresses
+loop:
+	for {
+		switch {
+		case aIdx == len(aAddrs) && bIdx == len(bAddrs):
+			// both lists are exhausted, we are done
+			break loop
+
+		case aIdx == len(aAddrs):
+			// a is exhausted, pick from b and continue
+			res.Addresses = append(res.Addresses, bAddrs[bIdx])
+			bIdx++
+			continue
+
+		case bIdx == len(bAddrs):
+			// b is exhausted, pick from a and continue
+			res.Addresses = append(res.Addresses, aAddrs[aIdx])
+			aIdx++
+			continue
+		}
+
+		// We have values left on both sides.
+		aVal := aAddrs[aIdx]
+		bVal := bAddrs[bIdx]
+
+		switch {
+		case aVal.Address == bVal.Address:
+			// update for same address, pick newer
+			if aVal.Expires > bVal.Expires {
+				res.Addresses = append(res.Addresses, aVal)
+			} else {
+				res.Addresses = append(res.Addresses, bVal)
+			}
 			aIdx++
 			bIdx++
-		case -1:
-			// a < b, keep a and move on
+
+		case aVal.Address < bVal.Address:
+			// a is smallest, pick it and continue
+			res.Addresses = append(res.Addresses, aVal)
 			aIdx++
-		case 1:
-			// a > b, insert b before a
-			a.Addresses = append(a.Addresses[:aIdx], append([]*discosrv.DatabaseAddress{b.Addresses[bIdx]}, a.Addresses[aIdx:]...)...)
+
+		default:
+			// b is smallest, pick it and continue
+			res.Addresses = append(res.Addresses, bVal)
 			bIdx++
 		}
 	}
-	if bIdx < len(b.Addresses) {
-		a.Addresses = append(a.Addresses, b.Addresses[bIdx:]...)
-	}
-
-	return a
+	return res
 }
 
 // expire returns the list of addresses after removing expired entries.
 // Expiration happen in place, so the slice given as the parameter is
-// destroyed. Internal order is preserved.
-func expire(addrs []*discosrv.DatabaseAddress, now time.Time) []*discosrv.DatabaseAddress {
-	cutoff := now.UnixNano()
-	naddrs := addrs[:0]
-	for i := range addrs {
-		if i > 0 && addrs[i].Address == addrs[i-1].Address {
-			// Skip duplicates
+// destroyed. Internal order is not preserved.
+func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
+	i := 0
+	for i < len(addrs) {
+		if addrs[i].Expires < now {
+			addrs = sliceutil.RemoveAndZero(addrs, i)
 			continue
 		}
-		if addrs[i].Expires >= cutoff {
-			naddrs = append(naddrs, addrs[i])
-		}
+		i++
 	}
-	if len(naddrs) == 0 {
-		return nil
-	}
-	return naddrs
+	return addrs
 }
 
-func Cmp(d, other *discosrv.DatabaseAddress) (n int) {
-	if c := cmp.Compare(d.Address, other.Address); c != 0 {
-		return c
-	}
-	return cmp.Compare(d.Expires, other.Expires)
+func sortedAddressCopy(addrs []DatabaseAddress) []DatabaseAddress {
+	sorted := make([]DatabaseAddress, len(addrs))
+	copy(sorted, addrs)
+	sort.Sort(databaseAddressOrder(sorted))
+	return sorted
 }
 
-func Equal(d, other *discosrv.DatabaseAddress) bool {
-	return d.Address == other.Address
+type databaseAddressOrder []DatabaseAddress
+
+func (s databaseAddressOrder) Less(a, b int) bool {
+	return s[a].Address < s[b].Address
+}
+
+func (s databaseAddressOrder) Swap(a, b int) {
+	s[a], s[b] = s[b], s[a]
+}
+
+func (s databaseAddressOrder) Len() int {
+	return len(s)
 }

cmd/stdiscosrv/database.pb.go

@@ -0,0 +1,847 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: database.proto
+
+package main
+
+import (
+	fmt "fmt"
+	_ "github.com/gogo/protobuf/gogoproto"
+	proto "github.com/gogo/protobuf/proto"
+	io "io"
+	math "math"
+	math_bits "math/bits"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+
+type DatabaseRecord struct {
+	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
+	Misses    int32             `protobuf:"varint,2,opt,name=misses,proto3" json:"misses,omitempty"`
+	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
+	Missed    int64             `protobuf:"varint,4,opt,name=missed,proto3" json:"missed,omitempty"`
+}
+
+func (m *DatabaseRecord) Reset()         { *m = DatabaseRecord{} }
+func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
+func (*DatabaseRecord) ProtoMessage()    {}
+func (*DatabaseRecord) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{0}
+}
+func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseRecord.Merge(m, src)
+}
+func (m *DatabaseRecord) XXX_Size() int {
+	return m.Size()
+}
+func (m *DatabaseRecord) XXX_DiscardUnknown() {
+	xxx_messageInfo_DatabaseRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DatabaseRecord proto.InternalMessageInfo
+
+type ReplicationRecord struct {
+	Key       string            `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+	Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses"`
+	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
+}
+
+func (m *ReplicationRecord) Reset()         { *m = ReplicationRecord{} }
+func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
+func (*ReplicationRecord) ProtoMessage()    {}
+func (*ReplicationRecord) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{1}
+}
+func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ReplicationRecord.Merge(m, src)
+}
+func (m *ReplicationRecord) XXX_Size() int {
+	return m.Size()
+}
+func (m *ReplicationRecord) XXX_DiscardUnknown() {
+	xxx_messageInfo_ReplicationRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ReplicationRecord proto.InternalMessageInfo
+
+type DatabaseAddress struct {
+	Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
+	Expires int64  `protobuf:"varint,2,opt,name=expires,proto3" json:"expires,omitempty"`
+}
+
+func (m *DatabaseAddress) Reset()         { *m = DatabaseAddress{} }
+func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
+func (*DatabaseAddress) ProtoMessage()    {}
+func (*DatabaseAddress) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{2}
+}
+func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseAddress.Merge(m, src)
+}
+func (m *DatabaseAddress) XXX_Size() int {
+	return m.Size()
+}
+func (m *DatabaseAddress) XXX_DiscardUnknown() {
+	xxx_messageInfo_DatabaseAddress.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DatabaseAddress proto.InternalMessageInfo
+
+func init() {
+	proto.RegisterType((*DatabaseRecord)(nil), "main.DatabaseRecord")
+	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
+	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
+}
+
+func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
+
+var fileDescriptor_b90fe3356ea5df07 = []byte{
+	// 270 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
+	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
+	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
+	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
+	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
+	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
+	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
+	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
+	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
+	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
+	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
+	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
+	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
+	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
+	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
+	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
+	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
+}
+
+func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Missed != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+		i--
+		dAtA[i] = 0x20
+	}
+	if m.Seen != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
+	}
+	if m.Misses != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+		i--
+		dAtA[i] = 0x10
+	}
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Seen != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
+	}
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x12
+		}
+	}
+	if len(m.Key) > 0 {
+		i -= len(m.Key)
+		copy(dAtA[i:], m.Key)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Expires != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
+		i--
+		dAtA[i] = 0x10
+	}
+	if len(m.Address) > 0 {
+		i -= len(m.Address)
+		copy(dAtA[i:], m.Address)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
+	offset -= sovDatabase(v)
+	base := offset
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return base
+}
+func (m *DatabaseRecord) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Addresses) > 0 {
+		for _, e := range m.Addresses {
+			l = e.Size()
+			n += 1 + l + sovDatabase(uint64(l))
+		}
+	}
+	if m.Misses != 0 {
+		n += 1 + sovDatabase(uint64(m.Misses))
+	}
+	if m.Seen != 0 {
+		n += 1 + sovDatabase(uint64(m.Seen))
+	}
+	if m.Missed != 0 {
+		n += 1 + sovDatabase(uint64(m.Missed))
+	}
+	return n
+}
+
+func (m *ReplicationRecord) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Key)
+	if l > 0 {
+		n += 1 + l + sovDatabase(uint64(l))
+	}
+	if len(m.Addresses) > 0 {
+		for _, e := range m.Addresses {
+			l = e.Size()
+			n += 1 + l + sovDatabase(uint64(l))
+		}
+	}
+	if m.Seen != 0 {
+		n += 1 + sovDatabase(uint64(m.Seen))
+	}
+	return n
+}
+
+func (m *DatabaseAddress) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Address)
+	if l > 0 {
+		n += 1 + l + sovDatabase(uint64(l))
+	}
+	if m.Expires != 0 {
+		n += 1 + sovDatabase(uint64(m.Expires))
+	}
+	return n
+}
+
+func sovDatabase(x uint64) (n int) {
+	return (math_bits.Len64(x|1) + 6) / 7
+}
+func sozDatabase(x uint64) (n int) {
+	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: DatabaseRecord: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: DatabaseRecord: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Addresses = append(m.Addresses, DatabaseAddress{})
+			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Misses", wireType)
+			}
+			m.Misses = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Misses |= int32(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
+			}
+			m.Seen = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Seen |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Missed", wireType)
+			}
+			m.Missed = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Missed |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ReplicationRecord: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ReplicationRecord: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Key = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Addresses = append(m.Addresses, DatabaseAddress{})
+			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
+			}
+			m.Seen = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Seen |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: DatabaseAddress: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: DatabaseAddress: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Address", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Address = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Expires", wireType)
+			}
+			m.Expires = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Expires |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipDatabase(dAtA []byte) (n int, err error) {
+	l := len(dAtA)
+	iNdEx := 0
+	depth := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if dAtA[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+		case 1:
+			iNdEx += 8
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if length < 0 {
+				return 0, ErrInvalidLengthDatabase
+			}
+			iNdEx += length
+		case 3:
+			depth++
+		case 4:
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupDatabase
+			}
+			depth--
+		case 5:
+			iNdEx += 4
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthDatabase
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
+	}
+	return 0, io.ErrUnexpectedEOF
+}
+
+var (
+	ErrInvalidLengthDatabase        = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowDatabase          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupDatabase = fmt.Errorf("proto: unexpected end of group")
+)

cmd/stdiscosrv/database.proto

@@ -0,0 +1,36 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+syntax = "proto3";
+
+package main;
+
+import "repos/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.goproto_unkeyed_all) = false;
+option (gogoproto.goproto_unrecognized_all) = false;
+option (gogoproto.goproto_sizecache_all) = false;
+
+message DatabaseRecord {
+    repeated DatabaseAddress addresses = 1 [(gogoproto.nullable) = false];
+    int32                    misses    = 2; // Number of lookups* without hits
+    int64                    seen      = 3; // Unix nanos, last device announce
+    int64                    missed    = 4; // Unix nanos, last* failed lookup
+}
+
+// *) Not every lookup results in a write, so may not be completely accurate
+
+message ReplicationRecord {
+    string                   key       = 1;
+    repeated DatabaseAddress addresses = 2 [(gogoproto.nullable) = false];
+    int64                    seen      = 3; // Unix nanos, last device announce
+}
+
+message DatabaseAddress {
+    string address = 1;
+    int64  expires = 2; // Unix nanos
+}

cmd/stdiscosrv/database_test.go

@@ -11,26 +11,29 @@ import (
 	"fmt"
 	"testing"
 	"time"
-
-	"github.com/syncthing/syncthing/internal/gen/discosrv"
-	"github.com/syncthing/syncthing/lib/protocol"
 )
 
 func TestDatabaseGetSet(t *testing.T) {
-	db := newInMemoryStore(t.TempDir(), 0, nil)
+	db, err := newMemoryLevelDBStore()
+	if err != nil {
+		t.Fatal(err)
+	}
 	ctx, cancel := context.WithCancel(context.Background())
 	go db.Serve(ctx)
 	defer cancel()
 
 	// Check missing record
 
-	rec, err := db.get(&protocol.EmptyDeviceID)
+	rec, err := db.get("abcd")
 	if err != nil {
 		t.Error("not found should not be an error")
 	}
 	if len(rec.Addresses) != 0 {
 		t.Error("addresses should be empty")
 	}
+	if rec.Misses != 0 {
+		t.Error("missing should be zero")
+	}
 
 	// Set up a clock
 
@@ -40,16 +43,16 @@ func TestDatabaseGetSet(t *testing.T) {
 
 	// Put a record
 
-	rec.Addresses = []*discosrv.DatabaseAddress{
+	rec.Addresses = []DatabaseAddress{
 		{Address: "tcp://1.2.3.4:5", Expires: tc.Now().Add(time.Minute).UnixNano()},
 	}
-	if err := db.put(&protocol.EmptyDeviceID, rec); err != nil {
+	if err := db.put("abcd", rec); err != nil {
 		t.Fatal(err)
 	}
 
 	// Verify it
 
-	rec, err = db.get(&protocol.EmptyDeviceID)
+	rec, err = db.get("abcd")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -66,16 +69,16 @@ func TestDatabaseGetSet(t *testing.T) {
 
 	tc.wind(30 * time.Second)
 
-	addrs := []*discosrv.DatabaseAddress{
+	addrs := []DatabaseAddress{
 		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
 	}
-	if err := db.merge(&protocol.EmptyDeviceID, addrs, tc.Now().UnixNano()); err != nil {
+	if err := db.merge("abcd", addrs, tc.Now().UnixNano()); err != nil {
 		t.Fatal(err)
 	}
 
 	// Verify it
 
-	rec, err = db.get(&protocol.EmptyDeviceID)
+	rec, err = db.get("abcd")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -98,7 +101,7 @@ func TestDatabaseGetSet(t *testing.T) {
 
 	// Verify it
 
-	rec, err = db.get(&protocol.EmptyDeviceID)
+	rec, err = db.get("abcd")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -111,18 +114,40 @@ func TestDatabaseGetSet(t *testing.T) {
 		t.Error("incorrect address")
 	}
 
-	// Set an address
+	// Put a record with misses
 
-	addrs = []*discosrv.DatabaseAddress{
-		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
-	}
-	if err := db.merge(&protocol.GlobalDeviceID, addrs, tc.Now().UnixNano()); err != nil {
+	rec = DatabaseRecord{Misses: 42, Missed: tc.Now().UnixNano()}
+	if err := db.put("efgh", rec); err != nil {
 		t.Fatal(err)
 	}
 
 	// Verify it
 
-	rec, err = db.get(&protocol.GlobalDeviceID)
+	rec, err = db.get("efgh")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 0 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have no addresses")
+	}
+	if rec.Misses != 42 {
+		t.Log(rec.Misses)
+		t.Error("incorrect misses")
+	}
+
+	// Set an address
+
+	addrs = []DatabaseAddress{
+		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
+	}
+	if err := db.merge("efgh", addrs, tc.Now().UnixNano()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it
+
+	rec, err = db.get("efgh")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -130,126 +155,48 @@ func TestDatabaseGetSet(t *testing.T) {
 		t.Log(rec.Addresses)
 		t.Fatal("should have one address")
 	}
+	if rec.Misses != 0 {
+		t.Log(rec.Misses)
+		t.Error("should have no misses")
+	}
 }
 
 func TestFilter(t *testing.T) {
 	// all cases are expired with t=10
 	cases := []struct {
-		a []*discosrv.DatabaseAddress
-		b []*discosrv.DatabaseAddress
+		a []DatabaseAddress
+		b []DatabaseAddress
 	}{
 		{
 			a: nil,
 			b: nil,
 		},
 		{
-			a: []*discosrv.DatabaseAddress{{Address: "a", Expires: 9}, {Address: "b", Expires: 9}, {Address: "c", Expires: 9}},
-			b: []*discosrv.DatabaseAddress{},
+			a: []DatabaseAddress{{Address: "a", Expires: 9}, {Address: "b", Expires: 9}, {Address: "c", Expires: 9}},
+			b: []DatabaseAddress{},
 		},
 		{
-			a: []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
-			b: []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
+			a: []DatabaseAddress{{Address: "a", Expires: 10}},
+			b: []DatabaseAddress{{Address: "a", Expires: 10}},
 		},
 		{
-			a: []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
-			b: []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
+			a: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
+			b: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
 		},
 		{
-			a: []*discosrv.DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
-			b: []*discosrv.DatabaseAddress{{Address: "b", Expires: 15}, {Address: "d", Expires: 15}},
+			a: []DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
+			b: []DatabaseAddress{{Address: "b", Expires: 15}, {Address: "d", Expires: 15}},
 		},
 	}
 
 	for _, tc := range cases {
-		res := expire(tc.a, time.Unix(0, 10))
+		res := expire(tc.a, 10)
 		if fmt.Sprint(res) != fmt.Sprint(tc.b) {
 			t.Errorf("Incorrect result %v, expected %v", res, tc.b)
 		}
 	}
 }
 
-func TestMerge(t *testing.T) {
-	cases := []struct {
-		a, b, res []*discosrv.DatabaseAddress
-	}{
-		{nil, nil, nil},
-		{
-			nil,
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
-		},
-		{
-			nil,
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
-		},
-		{
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 15}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 15}},
-		},
-		{
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
-			[]*discosrv.DatabaseAddress{{Address: "b", Expires: 15}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
-		},
-		{
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 15}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 15}},
-		},
-		{
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
-			[]*discosrv.DatabaseAddress{{Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
-		},
-		{
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
-			[]*discosrv.DatabaseAddress{{Address: "b", Expires: 5}, {Address: "c", Expires: 20}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
-		},
-		{
-			[]*discosrv.DatabaseAddress{{Address: "y", Expires: 10}, {Address: "z", Expires: 10}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "y", Expires: 10}, {Address: "z", Expires: 10}},
-		},
-		{
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "d", Expires: 10}},
-			[]*discosrv.DatabaseAddress{{Address: "b", Expires: 5}, {Address: "c", Expires: 20}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}, {Address: "d", Expires: 10}},
-		},
-	}
-
-	for _, tc := range cases {
-		rec := merge(&discosrv.DatabaseRecord{Addresses: tc.a}, &discosrv.DatabaseRecord{Addresses: tc.b})
-		if fmt.Sprint(rec.Addresses) != fmt.Sprint(tc.res) {
-			t.Errorf("Incorrect result %v, expected %v", rec.Addresses, tc.res)
-		}
-		rec = merge(&discosrv.DatabaseRecord{Addresses: tc.b}, &discosrv.DatabaseRecord{Addresses: tc.a})
-		if fmt.Sprint(rec.Addresses) != fmt.Sprint(tc.res) {
-			t.Errorf("Incorrect result %v, expected %v", rec.Addresses, tc.res)
-		}
-	}
-}
-
-func BenchmarkMergeEqual(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		ar := []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}}
-		br := []*discosrv.DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 10}}
-		res := merge(&discosrv.DatabaseRecord{Addresses: ar}, &discosrv.DatabaseRecord{Addresses: br})
-		if len(res.Addresses) != 2 {
-			b.Fatal("wrong length")
-		}
-		if res.Addresses[0].Address != "a" || res.Addresses[1].Address != "b" {
-			b.Fatal("wrong address")
-		}
-		if res.Addresses[0].Expires != 15 || res.Addresses[1].Expires != 15 {
-			b.Fatal("wrong expiry")
-		}
-	}
-	b.ReportAllocs() // should be zero per operation
-}
-
 type testClock struct {
 	now time.Time
 }

cmd/stdiscosrv/main.go

@@ -9,26 +9,22 @@ package main
 import (
 	"context"
 	"crypto/tls"
+	"flag"
 	"log"
+	"net"
 	"net/http"
-	_ "net/http/pprof"
 	"os"
-	"os/signal"
 	"runtime"
+	"strings"
 	"time"
 
-	"github.com/alecthomas/kong"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/thejerf/suture/v4"
-
-	"github.com/syncthing/syncthing/internal/blob"
-	"github.com/syncthing/syncthing/internal/blob/azureblob"
-	"github.com/syncthing/syncthing/internal/blob/s3"
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/tlsutil"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/thejerf/suture/v4"
 )
 
 const (
@@ -42,12 +38,17 @@ const (
 	errorRetryAfterSeconds = 1500
 	errorRetryFuzzSeconds  = 300
 
-	// Retry for not found is notFoundRetrySeenSeconds for records we have
-	// seen an announcement for (but it's not active right now) and
-	// notFoundRetryUnknownSeconds for records we have never seen (or not
-	// seen within the last week).
-	notFoundRetryUnknownMinSeconds = 60
-	notFoundRetryUnknownMaxSeconds = 3600
+	// Retry for not found is minSeconds + failures * incSeconds +
+	// random(fuzz), where failures is the number of consecutive lookups
+	// with no answer, up to maxSeconds. The fuzz is applied after capping
+	// to maxSeconds.
+	notFoundRetryMinSeconds  = 60
+	notFoundRetryMaxSeconds  = 3540
+	notFoundRetryIncSeconds  = 10
+	notFoundRetryFuzzSeconds = 60
+
+	// How often (in requests) we serialize the missed counter to database.
+	notFoundMissesWriteInterval = 10
 
 	httpReadTimeout    = 5 * time.Second
 	httpWriteTimeout   = 5 * time.Second
@@ -57,123 +58,164 @@ const (
 	replicationOutboxSize = 10000
 )
 
-var debug = false
-
-type CLI struct {
-	Cert                string  `group:"Listen" help:"Certificate file" default:"./cert.pem" env:"DISCOVERY_CERT_FILE"`
-	Key                 string  `group:"Listen" help:"Key file" default:"./key.pem" env:"DISCOVERY_KEY_FILE"`
-	HTTP                bool    `group:"Listen" help:"Listen on HTTP (behind an HTTPS proxy)" env:"DISCOVERY_HTTP"`
-	Compression         bool    `group:"Listen" help:"Enable GZIP compression of responses" env:"DISCOVERY_COMPRESSION"`
-	Listen              string  `group:"Listen" help:"Listen address" default:":8443" env:"DISCOVERY_LISTEN"`
-	MetricsListen       string  `group:"Listen" help:"Metrics listen address" env:"DISCOVERY_METRICS_LISTEN"`
-	DesiredNotFoundRate float64 `group:"Listen" help:"Desired maximum rate of not-found replies (/s)" default:"1000"`
-
-	DBDir           string        `group:"Database" help:"Database directory" default:"." env:"DISCOVERY_DB_DIR"`
-	DBFlushInterval time.Duration `group:"Database" help:"Interval between database flushes" default:"5m" env:"DISCOVERY_DB_FLUSH_INTERVAL"`
-
-	DBS3Endpoint    string `name:"db-s3-endpoint" group:"Database (S3 backup)" hidden:"true" help:"S3 endpoint for database" env:"DISCOVERY_DB_S3_ENDPOINT"`
-	DBS3Region      string `name:"db-s3-region" group:"Database (S3 backup)" hidden:"true" help:"S3 region for database" env:"DISCOVERY_DB_S3_REGION"`
-	DBS3Bucket      string `name:"db-s3-bucket" group:"Database (S3 backup)" hidden:"true" help:"S3 bucket for database" env:"DISCOVERY_DB_S3_BUCKET"`
-	DBS3AccessKeyID string `name:"db-s3-access-key-id" group:"Database (S3 backup)" hidden:"true" help:"S3 access key ID for database" env:"DISCOVERY_DB_S3_ACCESS_KEY_ID"`
-	DBS3SecretKey   string `name:"db-s3-secret-key" group:"Database (S3 backup)" hidden:"true" help:"S3 secret key for database" env:"DISCOVERY_DB_S3_SECRET_KEY"`
-
-	DBAzureBlobAccount   string `name:"db-azure-blob-account" env:"DISCOVERY_DB_AZUREBLOB_ACCOUNT"`
-	DBAzureBlobKey       string `name:"db-azure-blob-key" env:"DISCOVERY_DB_AZUREBLOB_KEY"`
-	DBAzureBlobContainer string `name:"db-azure-blob-container" env:"DISCOVERY_DB_AZUREBLOB_CONTAINER"`
-
-	AMQPAddress string `group:"AMQP replication" hidden:"true" help:"Address to AMQP broker" env:"DISCOVERY_AMQP_ADDRESS"`
-
-	Debug   bool `short:"d" help:"Print debug output" env:"DISCOVERY_DEBUG"`
-	Version bool `short:"v" help:"Print version and exit"`
+// These options make the database a little more optimized for writes, at
+// the expense of some memory usage and risk of losing writes in a (system)
+// crash.
+var levelDBOptions = &opt.Options{
+	NoSync:      true,
+	WriteBuffer: 32 << 20, // default 4<<20
 }
 
-func main() {
-	log.SetOutput(os.Stdout)
+var debug = false
 
-	var cli CLI
-	kong.Parse(&cli)
-	debug = cli.Debug
+func main() {
+	var listen string
+	var dir string
+	var metricsListen string
+	var replicationListen string
+	var replicationPeers string
+	var certFile string
+	var keyFile string
+	var replCertFile string
+	var replKeyFile string
+	var useHTTP bool
+	var largeDB bool
+
+	log.SetOutput(os.Stdout)
+	log.SetFlags(0)
+
+	flag.StringVar(&certFile, "cert", "./cert.pem", "Certificate file")
+	flag.StringVar(&keyFile, "key", "./key.pem", "Key file")
+	flag.StringVar(&dir, "db-dir", "./discovery.db", "Database directory")
+	flag.BoolVar(&debug, "debug", false, "Print debug output")
+	flag.BoolVar(&useHTTP, "http", false, "Listen on HTTP (behind an HTTPS proxy)")
+	flag.StringVar(&listen, "listen", ":8443", "Listen address")
+	flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
+	flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
+	flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
+	flag.StringVar(&replCertFile, "replication-cert", "", "Certificate file for replication")
+	flag.StringVar(&replKeyFile, "replication-key", "", "Key file for replication")
+	flag.BoolVar(&largeDB, "large-db", false, "Use larger database settings")
+	showVersion := flag.Bool("version", false, "Show version")
+	flag.Parse()
 
 	log.Println(build.LongVersionFor("stdiscosrv"))
-	if cli.Version {
+	if *showVersion {
 		return
 	}
 
 	buildInfo.WithLabelValues(build.Version, runtime.Version(), build.User, build.Date.UTC().Format("2006-01-02T15:04:05Z")).Set(1)
 
-	var cert tls.Certificate
-	if !cli.HTTP {
-		var err error
-		cert, err = tls.LoadX509KeyPair(cli.Cert, cli.Key)
-		if os.IsNotExist(err) {
-			log.Println("Failed to load keypair. Generating one, this might take a while...")
-			cert, err = tlsutil.NewCertificate(cli.Cert, cli.Key, "stdiscosrv", 20*365, false)
-			if err != nil {
-				log.Fatalln("Failed to generate X509 key pair:", err)
-			}
-		} else if err != nil {
-			log.Fatalln("Failed to load keypair:", err)
+	if largeDB {
+		levelDBOptions.BlockCacheCapacity = 64 << 20
+		levelDBOptions.BlockSize = 64 << 10
+		levelDBOptions.CompactionTableSize = 16 << 20
+		levelDBOptions.CompactionTableSizeMultiplier = 2.0
+		levelDBOptions.WriteBuffer = 64 << 20
+		levelDBOptions.CompactionL0Trigger = 8
+	}
+
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if os.IsNotExist(err) {
+		log.Println("Failed to load keypair. Generating one, this might take a while...")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 20*365)
+		if err != nil {
+			log.Fatalln("Failed to generate X509 key pair:", err)
+		}
+	} else if err != nil {
+		log.Fatalln("Failed to load keypair:", err)
+	}
+	devID := protocol.NewDeviceID(cert.Certificate[0])
+	log.Println("Server device ID is", devID)
+
+	replCert := cert
+	if replCertFile != "" && replKeyFile != "" {
+		replCert, err = tls.LoadX509KeyPair(replCertFile, replKeyFile)
+		if err != nil {
+			log.Fatalln("Failed to load replication keypair:", err)
+		}
+	}
+	replDevID := protocol.NewDeviceID(replCert.Certificate[0])
+	log.Println("Replication device ID is", replDevID)
+
+	// Parse the replication specs, if any.
+	var allowedReplicationPeers []protocol.DeviceID
+	var replicationDestinations []string
+	parts := strings.Split(replicationPeers, ",")
+	for _, part := range parts {
+		if part == "" {
+			continue
+		}
+
+		fields := strings.Split(part, "@")
+		switch len(fields) {
+		case 2:
+			// This is an id@address specification. Grab the address for the
+			// destination list. Try to resolve it once to catch obvious
+			// syntax errors here rather than having the sender service fail
+			// repeatedly later.
+			_, err := net.ResolveTCPAddr("tcp", fields[1])
+			if err != nil {
+				log.Fatalln("Resolving address:", err)
+			}
+			replicationDestinations = append(replicationDestinations, fields[1])
+			fallthrough // N.B.
+
+		case 1:
+			// The first part is always a device ID.
+			id, err := protocol.DeviceIDFromString(fields[0])
+			if err != nil {
+				log.Fatalln("Parsing device ID:", err)
+			}
+			if id == protocol.EmptyDeviceID {
+				log.Fatalf("Missing device ID for peer in %q", part)
+			}
+			allowedReplicationPeers = append(allowedReplicationPeers, id)
+
+		default:
+			log.Fatalln("Unrecognized replication spec:", part)
 		}
-		devID := protocol.NewDeviceID(cert.Certificate[0])
-		log.Println("Server device ID is", devID)
 	}
 
 	// Root of the service tree.
 	main := suture.New("main", suture.Spec{
 		PassThroughPanics: true,
-		Timeout:           2 * time.Minute,
 	})
 
-	// If configured, use blob storage for database backups.
-	var blobs blob.Store
-	var err error
-	if cli.DBS3Endpoint != "" {
-		blobs, err = s3.NewSession(cli.DBS3Endpoint, cli.DBS3Region, cli.DBS3Bucket, cli.DBS3AccessKeyID, cli.DBS3SecretKey)
-	} else if cli.DBAzureBlobAccount != "" {
-		blobs, err = azureblob.NewBlobStore(cli.DBAzureBlobAccount, cli.DBAzureBlobKey, cli.DBAzureBlobContainer)
-	}
-	if err != nil {
-		log.Fatalf("Failed to create blob store: %v", err)
-	}
-
 	// Start the database.
-	db := newInMemoryStore(cli.DBDir, cli.DBFlushInterval, blobs)
+	db, err := newLevelDBStore(dir)
+	if err != nil {
+		log.Fatalln("Open database:", err)
+	}
 	main.Add(db)
 
-	// If we have an AMQP broker for replication, start that
-	var repl replicator
-	if cli.AMQPAddress != "" {
-		clientID := rand.String(10)
-		kr := newAMQPReplicator(cli.AMQPAddress, clientID, db)
-		main.Add(kr)
-		repl = kr
+	// Start any replication senders.
+	var repl replicationMultiplexer
+	for _, dst := range replicationDestinations {
+		rs := newReplicationSender(dst, replCert, allowedReplicationPeers)
+		main.Add(rs)
+		repl = append(repl, rs)
+	}
+
+	// If we have replication configured, start the replication listener.
+	if len(allowedReplicationPeers) > 0 {
+		rl := newReplicationListener(replicationListen, replCert, allowedReplicationPeers, db)
+		main.Add(rl)
 	}
 
 	// Start the main API server.
-	qs := newAPISrv(cli.Listen, cert, db, repl, cli.HTTP, cli.Compression, cli.DesiredNotFoundRate)
+	qs := newAPISrv(listen, cert, db, repl, useHTTP)
 	main.Add(qs)
 
 	// If we have a metrics port configured, start a metrics handler.
-	if cli.MetricsListen != "" {
+	if metricsListen != "" {
 		go func() {
 			mux := http.NewServeMux()
 			mux.Handle("/metrics", promhttp.Handler())
-			log.Fatal(http.ListenAndServe(cli.MetricsListen, mux))
+			log.Fatal(http.ListenAndServe(metricsListen, mux))
 		}()
 	}
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	// Cancel on signal
-	signalChan := make(chan os.Signal, 1)
-	signal.Notify(signalChan, os.Interrupt)
-	go func() {
-		sig := <-signalChan
-		log.Printf("Received signal %s; shutting down", sig)
-		cancel()
-	}()
-
 	// Engage!
-	main.Serve(ctx)
+	main.Serve(context.Background())
 }

cmd/stdiscosrv/replication.go

@@ -0,0 +1,324 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/binary"
+	"fmt"
+	io "io"
+	"log"
+	"net"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+const (
+	replicationReadTimeout       = time.Minute
+	replicationWriteTimeout      = 30 * time.Second
+	replicationHeartbeatInterval = time.Second * 30
+)
+
+type replicator interface {
+	send(key string, addrs []DatabaseAddress, seen int64)
+}
+
+// a replicationSender tries to connect to the remote address and provide
+// them with a feed of replication updates.
+type replicationSender struct {
+	dst        string
+	cert       tls.Certificate // our certificate
+	allowedIDs []protocol.DeviceID
+	outbox     chan ReplicationRecord
+}
+
+func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
+	return &replicationSender{
+		dst:        dst,
+		cert:       cert,
+		allowedIDs: allowedIDs,
+		outbox:     make(chan ReplicationRecord, replicationOutboxSize),
+	}
+}
+
+func (s *replicationSender) Serve(ctx context.Context) error {
+	// Sleep a little at startup. Peers often restart at the same time, and
+	// this avoid the service failing and entering backoff state
+	// unnecessarily, while also reducing the reconnect rate to something
+	// reasonable by default.
+	time.Sleep(2 * time.Second)
+
+	tlsCfg := &tls.Config{
+		Certificates:       []tls.Certificate{s.cert},
+		MinVersion:         tls.VersionTLS12,
+		InsecureSkipVerify: true,
+	}
+
+	// Dial the TLS connection.
+	conn, err := tls.Dial("tcp", s.dst, tlsCfg)
+	if err != nil {
+		log.Println("Replication connect:", err)
+		return err
+	}
+	defer func() {
+		conn.SetWriteDeadline(time.Now().Add(time.Second))
+		conn.Close()
+	}()
+
+	// The replication stream is not especially latency sensitive, but it is
+	// quite a lot of data in small writes. Make it more efficient.
+	if tcpc, ok := conn.NetConn().(*net.TCPConn); ok {
+		_ = tcpc.SetNoDelay(false)
+	}
+
+	// Get the other side device ID.
+	remoteID, err := deviceID(conn)
+	if err != nil {
+		log.Println("Replication connect:", err)
+		return err
+	}
+
+	// Verify it's in the set of allowed device IDs.
+	if !deviceIDIn(remoteID, s.allowedIDs) {
+		log.Println("Replication connect: unexpected device ID:", remoteID)
+		return err
+	}
+
+	heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
+	defer heartBeatTicker.Stop()
+
+	// Send records.
+	buf := make([]byte, 1024)
+	for {
+		select {
+		case <-heartBeatTicker.C:
+			if len(s.outbox) > 0 {
+				// No need to send heartbeats if there are events/prevrious
+				// heartbeats to send, they will keep the connection alive.
+				continue
+			}
+			// Empty replication message is the heartbeat:
+			s.outbox <- ReplicationRecord{}
+
+		case rec := <-s.outbox:
+			// Buffer must hold record plus four bytes for size
+			size := rec.Size()
+			if len(buf) < size+4 {
+				buf = make([]byte, size+4)
+			}
+
+			// Record comes after the four bytes size
+			n, err := rec.MarshalTo(buf[4:])
+			if err != nil {
+				// odd to get an error here, but we haven't sent anything
+				// yet so it's not fatal
+				replicationSendsTotal.WithLabelValues("error").Inc()
+				log.Println("Replication marshal:", err)
+				continue
+			}
+			binary.BigEndian.PutUint32(buf, uint32(n))
+
+			// Send
+			conn.SetWriteDeadline(time.Now().Add(replicationWriteTimeout))
+			if _, err := conn.Write(buf[:4+n]); err != nil {
+				replicationSendsTotal.WithLabelValues("error").Inc()
+				log.Println("Replication write:", err)
+				// Yes, we are losing the replication event here.
+				return err
+			}
+			replicationSendsTotal.WithLabelValues("success").Inc()
+
+		case <-ctx.Done():
+			return nil
+		}
+	}
+}
+
+func (s *replicationSender) String() string {
+	return fmt.Sprintf("replicationSender(%q)", s.dst)
+}
+
+func (s *replicationSender) send(key string, ps []DatabaseAddress, _ int64) {
+	item := ReplicationRecord{
+		Key:       key,
+		Addresses: ps,
+	}
+
+	// The send should never block. The inbox is suitably buffered for at
+	// least a few seconds of stalls, which shouldn't happen in practice.
+	select {
+	case s.outbox <- item:
+	default:
+		replicationSendsTotal.WithLabelValues("drop").Inc()
+	}
+}
+
+// a replicationMultiplexer sends to multiple replicators
+type replicationMultiplexer []replicator
+
+func (m replicationMultiplexer) send(key string, ps []DatabaseAddress, seen int64) {
+	for _, s := range m {
+		// each send is nonblocking
+		s.send(key, ps, seen)
+	}
+}
+
+// replicationListener accepts incoming connections and reads replication
+// items from them. Incoming items are applied to the KV store.
+type replicationListener struct {
+	addr       string
+	cert       tls.Certificate
+	allowedIDs []protocol.DeviceID
+	db         database
+}
+
+func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
+	return &replicationListener{
+		addr:       addr,
+		cert:       cert,
+		allowedIDs: allowedIDs,
+		db:         db,
+	}
+}
+
+func (l *replicationListener) Serve(ctx context.Context) error {
+	tlsCfg := &tls.Config{
+		Certificates:       []tls.Certificate{l.cert},
+		ClientAuth:         tls.RequestClientCert,
+		MinVersion:         tls.VersionTLS12,
+		InsecureSkipVerify: true,
+	}
+
+	lst, err := tls.Listen("tcp", l.addr, tlsCfg)
+	if err != nil {
+		log.Println("Replication listen:", err)
+		return err
+	}
+	defer lst.Close()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+		}
+
+		// Accept a connection
+		conn, err := lst.Accept()
+		if err != nil {
+			log.Println("Replication accept:", err)
+			return err
+		}
+
+		// Figure out the other side device ID
+		remoteID, err := deviceID(conn.(*tls.Conn))
+		if err != nil {
+			log.Println("Replication accept:", err)
+			conn.SetWriteDeadline(time.Now().Add(time.Second))
+			conn.Close()
+			continue
+		}
+
+		// Verify it is in the set of allowed device IDs
+		if !deviceIDIn(remoteID, l.allowedIDs) {
+			log.Println("Replication accept: unexpected device ID:", remoteID)
+			conn.SetWriteDeadline(time.Now().Add(time.Second))
+			conn.Close()
+			continue
+		}
+
+		go l.handle(ctx, conn)
+	}
+}
+
+func (l *replicationListener) String() string {
+	return fmt.Sprintf("replicationListener(%q)", l.addr)
+}
+
+func (l *replicationListener) handle(ctx context.Context, conn net.Conn) {
+	defer func() {
+		conn.SetWriteDeadline(time.Now().Add(time.Second))
+		conn.Close()
+	}()
+
+	buf := make([]byte, 1024)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		conn.SetReadDeadline(time.Now().Add(replicationReadTimeout))
+
+		// First four bytes are the size
+		if _, err := io.ReadFull(conn, buf[:4]); err != nil {
+			log.Println("Replication read size:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			return
+		}
+
+		// Read the rest of the record
+		size := int(binary.BigEndian.Uint32(buf[:4]))
+		if len(buf) < size {
+			buf = make([]byte, size)
+		}
+
+		if size == 0 {
+			// Heartbeat, ignore
+			continue
+		}
+
+		if _, err := io.ReadFull(conn, buf[:size]); err != nil {
+			log.Println("Replication read record:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			return
+		}
+
+		// Unmarshal
+		var rec ReplicationRecord
+		if err := rec.Unmarshal(buf[:size]); err != nil {
+			log.Println("Replication unmarshal:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			continue
+		}
+
+		// Store
+		l.db.merge(rec.Key, rec.Addresses, rec.Seen)
+		replicationRecvsTotal.WithLabelValues("success").Inc()
+	}
+}
+
+func deviceID(conn *tls.Conn) (protocol.DeviceID, error) {
+	// Handshake may not be complete on the server side yet, which we need
+	// to get the client certificate.
+	if !conn.ConnectionState().HandshakeComplete {
+		if err := conn.Handshake(); err != nil {
+			return protocol.DeviceID{}, err
+		}
+	}
+
+	// We expect exactly one certificate.
+	certs := conn.ConnectionState().PeerCertificates
+	if len(certs) != 1 {
+		return protocol.DeviceID{}, fmt.Errorf("unexpected number of certificates (%d != 1)", len(certs))
+	}
+
+	return protocol.NewDeviceID(certs[0].Raw), nil
+}
+
+func deviceIDIn(id protocol.DeviceID, ids []protocol.DeviceID) bool {
+	for _, candidate := range ids {
+		if id == candidate {
+			return true
+		}
+	}
+	return false
+}

cmd/stdiscosrv/stats.go

@@ -7,7 +7,10 @@
 package main
 
 import (
+	"os"
+
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
 )
 
 var (
@@ -96,28 +99,13 @@ var (
 			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
 		}, []string{"operation"})
 
-	databaseWriteSeconds = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "database_write_seconds",
-			Help:      "Time spent writing the database.",
-		})
-	databaseLastWritten = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "database_last_written",
-			Help:      "Timestamp of the last successful database write.",
-		})
-
-	retryAfterLevel = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "retry_after_seconds",
-			Help:      "Retry-After header value in seconds.",
-		}, []string{"name"})
+	retryAfterHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace: "syncthing",
+		Subsystem: "discovery",
+		Name:      "retry_after_seconds",
+		Help:      "Retry-After header value in seconds.",
+		Buckets:   prometheus.ExponentialBuckets(60, 2, 7), // 60, 120, 240, 480, 960, 1920, 3840
+	})
 )
 
 const (
@@ -138,6 +126,16 @@ func init() {
 		replicationSendsTotal, replicationRecvsTotal,
 		databaseKeys, databaseStatisticsSeconds,
 		databaseOperations, databaseOperationSeconds,
-		databaseWriteSeconds, databaseLastWritten,
-		retryAfterLevel)
+		retryAfterHistogram)
+
+	processCollectorOpts := collectors.ProcessCollectorOpts{
+		Namespace: "syncthing_discovery",
+		PidFn: func() (int, error) {
+			return os.Getpid(), nil
+		},
+	}
+
+	prometheus.MustRegister(
+		collectors.NewProcessCollector(processCollectorOpts),
+	)
 }

cmd/stfileinfo/main.go

@@ -72,7 +72,7 @@ func main() {
 		if *standardBlocks || blockSize < protocol.MinBlockSize {
 			blockSize = protocol.BlockSize(fi.Size())
 		}
-		bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil)
+		bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/strelaypoolsrv/README.md

@@ -10,7 +10,7 @@ to NAT or firewall issues.
 
 There is very little reason why you'd want to run this yourself, as
 `relaypoolsrv` is just used for announcement and lookup of public relay
-servers. If you are looking to set up a private or a public relay, please
+servers. If you are looking to setup a private or a public relay, please
 check the documentation for
 [relaysrv](https://github.com/syncthing/relaysrv), which also explains how
 to join the default public pool.
@@ -21,3 +21,4 @@ See `relaypoolsrv -help` for configuration options.
 
 [oschwald/geoip2-golang](https://github.com/oschwald/geoip2-golang), [oschwald/maxminddb-golang](https://github.com/oschwald/maxminddb-golang), Copyright (C) 2015 [Gregory J. Oschwald](mailto:oschwald@gmail.com).
 
+[lib/pq](https://github.com/lib/pq)</a>, Copyright (C) 2011-2013 'pq' Contributors Portions Copyright (C) 2011 Blake Mizerany.

cmd/strelaypoolsrv/auto/doc.go

@@ -4,7 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//go:generate go run ../../../../script/genassets.go -o gui.files.go ../gui
+//go:generate go run ../../../script/genassets.go -o gui.files.go ../gui
 
 // Package auto contains auto generated files for web assets.
 package auto

cmd/strelaypoolsrv/gui/index.html

@@ -259,7 +259,7 @@
       return a.value > b.value ? 1 : -1;
     }
 
-    $http.get("/endpoint/full").then(function(response) {
+    $http.get("/endpoint").then(function(response) {
       $scope.relays = response.data.relays;
 
       angular.forEach($scope.relays, function(relay) {
@@ -338,7 +338,7 @@
         relay.showMarker = function() {
           relay.marker.openPopup();
         }
-
+        
         relay.hideMarker = function() {
           relay.marker.closePopup();
         }
@@ -347,7 +347,7 @@
 
     function addCircleToMap(relay) {
       console.log(relay.location.latitude)
-      L.circle([relay.location.latitude, relay.location.longitude],
+      L.circle([relay.location.latitude, relay.location.longitude], 
         {
           radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000,
           color: "FF0000",

cmd/strelaypoolsrv/main.go

@@ -17,21 +17,21 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"sync"
 	"sync/atomic"
 	"time"
 
 	lru "github.com/hashicorp/golang-lru/v2"
+	"github.com/oschwald/geoip2-golang"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-
-	"github.com/syncthing/syncthing/cmd/infra/strelaypoolsrv/auto"
+	"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
 	"github.com/syncthing/syncthing/lib/assets"
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
-	"github.com/syncthing/syncthing/lib/geoip"
+	"github.com/syncthing/syncthing/lib/httpcache"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/relay/client"
+	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 )
 
@@ -51,10 +51,6 @@ type relay struct {
 	StatsRetrieved time.Time `json:"statsRetrieved"`
 }
 
-type relayShort struct {
-	URL string `json:"url"`
-}
-
 type stats struct {
 	StartTime          time.Time `json:"startTime"`
 	UptimeSeconds      int       `json:"uptimeSeconds"`
@@ -99,23 +95,20 @@ var (
 	testCert          tls.Certificate
 	knownRelaysFile   = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
 	listen            = ":80"
-	metricsListen     = ":8081"
 	dir               string
 	evictionTime      = time.Hour
 	debug             bool
 	permRelaysFile    string
 	ipHeader          string
+	geoipPath         string
 	proto             string
 	statsRefresh      = time.Minute
 	requestQueueLen   = 64
 	requestProcessors = 8
-	geoipLicenseKey   = os.Getenv("GEOIP_LICENSE_KEY")
-	geoipAccountID, _ = strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID"))
-	maxRelaysReturned = 100
 
 	requests chan request
 
-	mut             sync.RWMutex
+	mut             = sync.NewRWMutex()
 	knownRelays     = make([]*relay, 0)
 	permanentRelays = make([]*relay, 0)
 	evictionTimers  = make(map[string]*time.Timer)
@@ -131,46 +124,40 @@ func main() {
 	log.SetFlags(log.Lshortfile)
 
 	flag.StringVar(&listen, "listen", listen, "Listen address")
-	flag.StringVar(&metricsListen, "metrics-listen", metricsListen, "Metrics listen address")
 	flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
 	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
 	flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
 	flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
 	flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays")
 	flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
+	flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
 	flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
 	flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
 	flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
 	flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
-	flag.StringVar(&geoipLicenseKey, "geoip-license-key", geoipLicenseKey, "License key for GeoIP database")
-	flag.IntVar(&maxRelaysReturned, "max-relays-returned", maxRelaysReturned, "Maximum number of relays returned for a normal endpoint query")
 
 	flag.Parse()
 
 	requests = make(chan request, requestQueueLen)
-	geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), geoipAccountID, geoipLicenseKey, os.TempDir())
-	if err != nil {
-		log.Fatalln("Failed to create GeoIP provider:", err)
-	}
-	go geoip.Serve(context.TODO())
 
 	var listener net.Listener
+	var err error
 
 	if permRelaysFile != "" {
-		permanentRelays = loadRelays(permRelaysFile, geoip)
+		permanentRelays = loadRelays(permRelaysFile)
 	}
 
 	testCert = createTestCertificate()
 
-	for range requestProcessors {
-		go requestProcessor(geoip)
+	for i := 0; i < requestProcessors; i++ {
+		go requestProcessor()
 	}
 
 	// Load relays from cache in the background.
 	// Load them in a serial fashion to make sure any genuine requests
 	// are not dropped.
 	go func() {
-		for _, relay := range loadRelays(knownRelaysFile, geoip) {
+		for _, relay := range loadRelays(knownRelaysFile) {
 			resultChan := make(chan result)
 			requests <- request{relay, resultChan, nil}
 			result := <-resultChan
@@ -180,7 +167,7 @@ func main() {
 				relayTestsTotal.WithLabelValues("success").Inc()
 			}
 		}
-		// Run the stats refresher once the relays are loaded.
+		// Run the the stats refresher once the relays are loaded.
 		statsRefresher(statsRefresh)
 	}()
 
@@ -226,40 +213,15 @@ func main() {
 		log.Fatalln("listen:", err)
 	}
 
-	if metricsListen != "" {
-		mmux := http.NewServeMux()
-		mmux.HandleFunc("/metrics", handleMetrics)
-		go func() {
-			if err := http.ListenAndServe(metricsListen, mmux); err != nil {
-				log.Fatalln("HTTP serve metrics:", err)
-			}
-		}()
-	}
-
-	getMux := http.NewServeMux()
-	getMux.HandleFunc("/", handleAssets)
-	getMux.HandleFunc("/endpoint", withAPIMetrics(handleEndpointShort))
-	getMux.HandleFunc("/endpoint/full", withAPIMetrics(handleEndpointFull))
-
-	postMux := http.NewServeMux()
-	postMux.HandleFunc("/endpoint", withAPIMetrics(handleRegister))
-
-	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		switch r.Method {
-		case http.MethodGet, http.MethodHead, http.MethodOptions:
-			getMux.ServeHTTP(w, r)
-		case http.MethodPost:
-			postMux.ServeHTTP(w, r)
-		default:
-			http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
-		}
-	})
+	handler := http.NewServeMux()
+	handler.HandleFunc("/", handleAssets)
+	handler.Handle("/endpoint", httpcache.SinglePath(http.HandlerFunc(handleRequest), 15*time.Second))
+	handler.HandleFunc("/metrics", handleMetrics)
 
 	srv := http.Server{
 		Handler:     handler,
 		ReadTimeout: 10 * time.Second,
 	}
-	srv.SetKeepAlivesEnabled(false)
 
 	err = srv.Serve(listener)
 	if err != nil {
@@ -293,24 +255,39 @@ func handleAssets(w http.ResponseWriter, r *http.Request) {
 	assets.Serve(w, r, as)
 }
 
-func withAPIMetrics(next http.HandlerFunc) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
-		w = NewLoggingResponseWriter(w)
-		defer func() {
-			timer.ObserveDuration()
-			lw := w.(*loggingResponseWriter)
-			apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
-		}()
-		next(w, r)
+func handleRequest(w http.ResponseWriter, r *http.Request) {
+	timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
+
+	w = NewLoggingResponseWriter(w)
+	defer func() {
+		timer.ObserveDuration()
+		lw := w.(*loggingResponseWriter)
+		apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
+	}()
+
+	if ipHeader != "" {
+		hdr := r.Header.Get(ipHeader)
+		fields := strings.Split(hdr, ",")
+		if len(fields) > 0 {
+			r.RemoteAddr = strings.TrimSpace(fields[len(fields)-1])
+		}
+	}
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	switch r.Method {
+	case "GET":
+		handleGetRequest(w, r)
+	case "POST":
+		handlePostRequest(w, r)
+	default:
+		if debug {
+			log.Println("Unhandled HTTP method", r.Method)
+		}
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 	}
 }
 
-// handleEndpointFull returns the relay list with full metadata and
-// statistics. Large, and expensive.
-func handleEndpointFull(rw http.ResponseWriter, r *http.Request) {
+func handleGetRequest(rw http.ResponseWriter, r *http.Request) {
 	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
-	rw.Header().Set("Access-Control-Allow-Origin", "*")
 
 	mut.RLock()
 	relays := make([]*relay, len(permanentRelays)+len(knownRelays))
@@ -318,42 +295,17 @@ func handleEndpointFull(rw http.ResponseWriter, r *http.Request) {
 	copy(relays[n:], knownRelays)
 	mut.RUnlock()
 
+	// Shuffle
+	rand.Shuffle(relays)
+
 	_ = json.NewEncoder(rw).Encode(map[string][]*relay{
 		"relays": relays,
 	})
 }
 
-// handleEndpointShort returns the relay list with only the URL.
-func handleEndpointShort(rw http.ResponseWriter, r *http.Request) {
-	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
-	rw.Header().Set("Access-Control-Allow-Origin", "*")
-
-	mut.RLock()
-	relays := make([]relayShort, 0, len(permanentRelays)+len(knownRelays))
-	for _, r := range append(permanentRelays, knownRelays...) {
-		relays = append(relays, relayShort{URL: slimURL(r.URL)})
-	}
-	mut.RUnlock()
-	if len(relays) > maxRelaysReturned {
-		rand.Shuffle(relays)
-		relays = relays[:maxRelaysReturned]
-	}
-
-	_ = json.NewEncoder(rw).Encode(map[string][]relayShort{
-		"relays": relays,
-	})
-}
-
-func handleRegister(w http.ResponseWriter, r *http.Request) {
+func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 	// Get the IP address of the client
 	rhost := r.RemoteAddr
-	if ipHeader != "" {
-		hdr := r.Header.Get(ipHeader)
-		fields := strings.Split(hdr, ",")
-		if len(fields) > 0 {
-			rhost = strings.TrimSpace(fields[len(fields)-1])
-		}
-	}
 	if host, _, err := net.SplitHostPort(rhost); err == nil {
 		rhost = host
 	}
@@ -473,19 +425,19 @@ func handleRegister(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func requestProcessor(geoip *geoip.Provider) {
+func requestProcessor() {
 	for request := range requests {
 		if request.queueTimer != nil {
 			request.queueTimer.ObserveDuration()
 		}
 
 		timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test"))
-		handleRelayTest(request, geoip)
+		handleRelayTest(request)
 		timer.ObserveDuration()
 	}
 }
 
-func handleRelayTest(request request, geoip *geoip.Provider) {
+func handleRelayTest(request request) {
 	if debug {
 		log.Println("Request for", request.relay)
 	}
@@ -498,7 +450,7 @@ func handleRelayTest(request request, geoip *geoip.Provider) {
 	}
 
 	stats := fetchStats(request.relay)
-	location := getLocation(request.relay.uri.Host, geoip)
+	location := getLocation(request.relay.uri.Host)
 
 	mut.Lock()
 	if stats != nil {
@@ -571,7 +523,7 @@ func evict(relay *relay) func() {
 	}
 }
 
-func loadRelays(file string, geoip *geoip.Provider) []*relay {
+func loadRelays(file string) []*relay {
 	content, err := os.ReadFile(file)
 	if err != nil {
 		log.Println("Failed to load relays: " + err.Error())
@@ -595,7 +547,7 @@ func loadRelays(file string, geoip *geoip.Provider) []*relay {
 
 		relays = append(relays, &relay{
 			URL:      line,
-			Location: getLocation(uri.Host, geoip),
+			Location: getLocation(uri.Host),
 			uri:      uri,
 		})
 		if debug {
@@ -620,7 +572,7 @@ func createTestCertificate() tls.Certificate {
 	}
 
 	certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
-	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 20*365, false)
+	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 20*365)
 	if err != nil {
 		log.Fatalln("Failed to create test X509 key pair:", err)
 	}
@@ -628,16 +580,21 @@ func createTestCertificate() tls.Certificate {
 	return cert
 }
 
-func getLocation(host string, geoip *geoip.Provider) location {
+func getLocation(host string) location {
 	timer := prometheus.NewTimer(locationLookupSeconds)
 	defer timer.ObserveDuration()
+	db, err := geoip2.Open(geoipPath)
+	if err != nil {
+		return location{}
+	}
+	defer db.Close()
 
 	addr, err := net.ResolveTCPAddr("tcp", host)
 	if err != nil {
 		return location{}
 	}
 
-	city, err := geoip.City(addr.IP)
+	city, err := db.City(addr.IP)
 	if err != nil {
 		return location{}
 	}
@@ -653,7 +610,6 @@ func getLocation(host string, geoip *geoip.Provider) location {
 
 type loggingResponseWriter struct {
 	http.ResponseWriter
-
 	statusCode int
 }
 
@@ -704,16 +660,3 @@ func (b *errorTracker) IsBlocked(host string) bool {
 	}
 	return false
 }
-
-func slimURL(u string) string {
-	p, err := url.Parse(u)
-	if err != nil {
-		return u
-	}
-	newQuery := url.Values{}
-	if id := p.Query().Get("id"); id != "" {
-		newQuery.Set("id", id)
-	}
-	p.RawQuery = newQuery.Encode()
-	return p.String()
-}

cmd/strelaypoolsrv/main_test.go

@@ -13,6 +13,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"strings"
+	"sync"
 	"testing"
 )
 
@@ -27,6 +28,8 @@ func init() {
 		{URL: "known2"},
 		{URL: "known3"},
 	}
+
+	mut = new(sync.RWMutex)
 }
 
 // Regression test: handleGetRequest should not modify permanentRelays.
@@ -39,7 +42,7 @@ func TestHandleGetRequest(t *testing.T) {
 
 	w := httptest.NewRecorder()
 	w.Body = new(bytes.Buffer)
-	handleEndpointFull(w, httptest.NewRequest("GET", "/", nil))
+	handleGetRequest(w, httptest.NewRequest("GET", "/", nil))
 
 	result := make(map[string][]*relay)
 	err := json.NewDecoder(w.Body).Decode(&result)
@@ -89,18 +92,3 @@ func TestCanonicalizeQueryValues(t *testing.T) {
 		t.Errorf("expected %q, got %q", exp, str)
 	}
 }
-
-func TestSlimURL(t *testing.T) {
-	cases := []struct {
-		in, out string
-	}{
-		{"http://example.com/", "http://example.com/"},
-		{"relay://192.0.2.42:22067/?globalLimitBps=0&id=EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M&networkTimeout=2m0s&pingInterval=1m0s&providedBy=Test&sessionLimitBps=0&statusAddr=%3A22070", "relay://192.0.2.42:22067/?id=EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M"},
-	}
-
-	for _, c := range cases {
-		if got := slimURL(c.in); got != c.out {
-			t.Errorf("expected %q, got %q", c.out, got)
-		}
-	}
-}

cmd/strelaypoolsrv/stats.go

@@ -6,12 +6,27 @@ import (
 	"encoding/json"
 	"net"
 	"net/http"
-	"sync"
+	"os"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/syncthing/syncthing/lib/sync"
 )
 
+func init() {
+	processCollectorOpts := collectors.ProcessCollectorOpts{
+		Namespace: "syncthing_relaypoolsrv",
+		PidFn: func() (int, error) {
+			return os.Getpid(), nil
+		},
+	}
+
+	prometheus.MustRegister(
+		collectors.NewProcessCollector(processCollectorOpts),
+	)
+}
+
 var (
 	statusClient = http.Client{
 		Timeout: 5 * time.Second,
@@ -104,7 +119,7 @@ func refreshStats() {
 	mut.RUnlock()
 
 	now := time.Now()
-	var wg sync.WaitGroup
+	wg := sync.NewWaitGroup()
 
 	results := make(chan statsFetchResult, len(relays))
 	for _, rel := range relays {
@@ -173,7 +188,7 @@ func fetchStats(relay *relay) *stats {
 
 	var stats stats
 
-	if err := json.NewDecoder(response.Body).Decode(&stats); err != nil {
+	if json.NewDecoder(response.Body).Decode(&stats); err != nil {
 		return nil
 	}
 	return &stats

cmd/strelaysrv/listener.go

@@ -12,8 +12,9 @@ import (
 	"time"
 
 	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/relay/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
+
+	"github.com/syncthing/syncthing/lib/relay/protocol"
 )
 
 var (
@@ -184,7 +185,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token strin
 					continue
 				}
 				// requestedPeer is the server, id is the client
-				ses := newSession(requestedPeer, id, sessionLimitBps, globalLimiter)
+				ses := newSession(requestedPeer, id, sessionLimiter, globalLimiter)
 
 				go ses.Serve()
 

cmd/strelaysrv/main.go

@@ -14,14 +14,11 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"sync/atomic"
 	"syscall"
 	"time"
 
-	"golang.org/x/time/rate"
-
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/config"
@@ -33,6 +30,7 @@ import (
 	"github.com/syncthing/syncthing/lib/relay/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	_ "github.com/syncthing/syncthing/lib/upnp"
+	"golang.org/x/time/rate"
 )
 
 var (
@@ -52,6 +50,7 @@ var (
 	globalLimitBps    int
 	overLimit         atomic.Bool
 	descriptorLimit   int64
+	sessionLimiter    *rate.Limiter
 	globalLimiter     *rate.Limiter
 	networkBufferSize int
 
@@ -158,7 +157,7 @@ func main() {
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 20*365, false)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
@@ -228,6 +227,9 @@ func main() {
 		}
 	}
 
+	if sessionLimitBps > 0 {
+		sessionLimiter = rate.NewLimiter(rate.Limit(sessionLimitBps), 2*sessionLimitBps)
+	}
 	if globalLimitBps > 0 {
 		globalLimiter = rate.NewLimiter(rate.Limit(globalLimitBps), 2*globalLimitBps)
 	}
@@ -248,10 +250,10 @@ func main() {
 	query.Set("pingInterval", pingInterval.String())
 	query.Set("networkTimeout", networkTimeout.String())
 	if sessionLimitBps > 0 {
-		query.Set("sessionLimitBps", strconv.Itoa(sessionLimitBps))
+		query.Set("sessionLimitBps", fmt.Sprint(sessionLimitBps))
 	}
 	if globalLimitBps > 0 {
-		query.Set("globalLimitBps", strconv.Itoa(globalLimitBps))
+		query.Set("globalLimitBps", fmt.Sprint(globalLimitBps))
 	}
 	if statusAddr != "" {
 		query.Set("statusAddr", statusAddr)

cmd/strelaysrv/session.go

@@ -27,7 +27,7 @@ var (
 	bytesProxied    atomic.Int64
 )
 
-func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionLimitBps int, globalRateLimit *rate.Limiter) *session {
+func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionRateLimit, globalRateLimit *rate.Limiter) *session {
 	serverkey := make([]byte, 32)
 	_, err := rand.Read(serverkey)
 	if err != nil {
@@ -40,17 +40,12 @@ func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionLimitBps i
 		return nil
 	}
 
-	var sessionRateLimit *rate.Limiter
-	if sessionLimitBps > 0 {
-		sessionRateLimit = rate.NewLimiter(rate.Limit(sessionLimitBps), 2*sessionLimitBps)
-	}
 	ses := &session{
 		serverkey: serverkey,
 		serverid:  serverid,
 		clientkey: clientkey,
 		clientid:  clientid,
 		rateLimit: makeRateLimitFunc(sessionRateLimit, globalRateLimit),
-		limiter:   sessionRateLimit,
 		connsChan: make(chan net.Conn),
 		conns:     make([]net.Conn, 0, 2),
 	}
@@ -73,6 +68,7 @@ func findSession(key string) *session {
 	ses, ok := pendingSessions[key]
 	if !ok {
 		return nil
+
 	}
 	delete(pendingSessions, key)
 	return ses
@@ -114,7 +110,6 @@ type session struct {
 	clientid  syncthingprotocol.DeviceID
 
 	rateLimit func(bytes int)
-	limiter   *rate.Limiter
 
 	connsChan chan net.Conn
 	conns     []net.Conn

cmd/strelaysrv/status.go

@@ -122,7 +122,7 @@ func (r *rateCalculator) updateRates(interval time.Duration) {
 
 func (r *rateCalculator) rate(periods int) int64 {
 	var tot int64
-	for i := range periods {
+	for i := 0; i < periods; i++ {
 		tot += r.rates[i]
 	}
 	return tot / int64(periods)

cmd/strelaysrv/testutil/main.go

@@ -6,7 +6,6 @@ import (
 	"bufio"
 	"context"
 	"crypto/tls"
-	"errors"
 	"flag"
 	"log"
 	"net"
@@ -134,8 +133,7 @@ func connectToStdio(stdin <-chan string, conn net.Conn) {
 		conn.SetReadDeadline(time.Now().Add(time.Millisecond))
 		n, err := conn.Read(buf[0:])
 		if err != nil {
-			var nerr net.Error
-			ok := errors.As(err, &nerr)
+			nerr, ok := err.(net.Error)
 			if !ok || !nerr.Timeout() {
 				log.Println(err)
 				return

cmd/strelaysrv/utils.go

@@ -10,7 +10,7 @@ import (
 func setTCPOptions(conn net.Conn) error {
 	tcpConn, ok := conn.(*net.TCPConn)
 	if !ok {
-		return errors.New("not a TCP connection")
+		return errors.New("Not a TCP connection")
 	}
 	if err := tcpConn.SetLinger(0); err != nil {
 		return err

cmd/stupgrades/main.go

@@ -0,0 +1,149 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/alecthomas/kong"
+	_ "github.com/syncthing/syncthing/lib/automaxprocs"
+	"github.com/syncthing/syncthing/lib/httpcache"
+	"github.com/syncthing/syncthing/lib/upgrade"
+)
+
+type cli struct {
+	Listen    string        `default:":8080" help:"Listen address"`
+	URL       string        `short:"u" default:"https://api.github.com/repos/syncthing/syncthing/releases?per_page=25" help:"GitHub releases url"`
+	Forward   []string      `short:"f" help:"Forwarded pages, format: /path->https://example/com/url"`
+	CacheTime time.Duration `default:"15m" help:"Cache time"`
+}
+
+func main() {
+	var params cli
+	kong.Parse(&params)
+	if err := server(&params); err != nil {
+		fmt.Printf("Error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func server(params *cli) error {
+	http.Handle("/meta.json", httpcache.SinglePath(&githubReleases{url: params.URL}, params.CacheTime))
+
+	for _, fwd := range params.Forward {
+		path, url, ok := strings.Cut(fwd, "->")
+		if !ok {
+			return fmt.Errorf("invalid forward: %q", fwd)
+		}
+		http.Handle(path, httpcache.SinglePath(&proxy{url: url}, params.CacheTime))
+	}
+
+	return http.ListenAndServe(params.Listen, nil)
+}
+
+type githubReleases struct {
+	url string
+}
+
+func (p *githubReleases) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	log.Println("Fetching", p.url)
+	rels := upgrade.FetchLatestReleases(p.url, "")
+	if rels == nil {
+		http.Error(w, "no releases", http.StatusInternalServerError)
+		return
+	}
+
+	sort.Sort(upgrade.SortByRelease(rels))
+	rels = filterForLatest(rels)
+
+	// Move the URL used for browser downloads to the URL field, and remove
+	// the browser URL field. This avoids going via the GitHub API for
+	// downloads, since Syncthing uses the URL field.
+	for _, rel := range rels {
+		for j, asset := range rel.Assets {
+			rel.Assets[j].URL = asset.BrowserURL
+			rel.Assets[j].BrowserURL = ""
+		}
+	}
+
+	buf := new(bytes.Buffer)
+	_ = json.NewEncoder(buf).Encode(rels)
+
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Access-Control-Allow-Methods", "GET")
+	w.Write(buf.Bytes())
+}
+
+type proxy struct {
+	url string
+}
+
+func (p *proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	log.Println("Fetching", p.url)
+	req, err := http.NewRequestWithContext(req.Context(), http.MethodGet, p.url, nil)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	defer resp.Body.Close()
+
+	ct := resp.Header.Get("Content-Type")
+	w.Header().Set("Content-Type", ct)
+	if resp.StatusCode == http.StatusOK {
+		w.Header().Set("Cache-Control", "public, max-age=900")
+		w.Header().Set("Access-Control-Allow-Origin", "*")
+		w.Header().Set("Access-Control-Allow-Methods", "GET")
+	}
+	w.WriteHeader(resp.StatusCode)
+	if strings.HasPrefix(ct, "application/json") {
+		// Special JSON handling; clean it up a bit.
+		var v interface{}
+		if err := json.NewDecoder(resp.Body).Decode(&v); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		_ = json.NewEncoder(w).Encode(v)
+	} else {
+		_, _ = io.Copy(w, resp.Body)
+	}
+}
+
+// filterForLatest returns the latest stable and prerelease only. If the
+// stable version is newer (comes first in the list) there is no need to go
+// looking for a prerelease at all.
+func filterForLatest(rels []upgrade.Release) []upgrade.Release {
+	var filtered []upgrade.Release
+	var havePre bool
+	for _, rel := range rels {
+		if !rel.Prerelease {
+			// We found a stable version, we're good now.
+			filtered = append(filtered, rel)
+			break
+		}
+		if rel.Prerelease && !havePre {
+			// We remember the first prerelease we find.
+			filtered = append(filtered, rel)
+			havePre = true
+		}
+	}
+	return filtered
+}

cmd/stwatchfile/main.go

@@ -7,7 +7,6 @@
 package main
 
 import (
-	"crypto/sha256"
 	"flag"
 	"fmt"
 	"io"
@@ -15,6 +14,7 @@ import (
 	"time"
 
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 func main() {

cmd/syncthing/blockprof.go

@@ -8,14 +8,11 @@ package main
 
 import (
 	"fmt"
-	"log/slog"
 	"os"
 	"runtime"
 	"runtime/pprof"
 	"syscall"
 	"time"
-
-	"github.com/syncthing/syncthing/internal/slogutil"
 )
 
 func startBlockProfiler() {
@@ -23,10 +20,10 @@ func startBlockProfiler() {
 	if profiler == nil {
 		panic("Couldn't find block profiler")
 	}
-	slog.Debug("Starting block profiling")
+	l.Debugln("Starting block profiling")
 	go func() {
 		err := saveBlockingProfiles(profiler) // Only returns on error
-		slog.Error("Block profiler failed", slogutil.Error(err))
+		l.Warnln("Block profiler failed:", err)
 		panic("Block profiler failed")
 	}()
 }

cmd/syncthing/cli/client.go

@@ -32,7 +32,6 @@ type APIClient interface {
 
 type apiClient struct {
 	http.Client
-
 	cfg    config.GUIConfiguration
 	apikey string
 }
@@ -92,11 +91,11 @@ func loadGUIConfig() (config.GUIConfiguration, error) {
 	guiCfg := cfg.GUI()
 
 	if guiCfg.Address() == "" {
-		return config.GUIConfiguration{}, errors.New("could not find GUI Address")
+		return config.GUIConfiguration{}, errors.New("Could not find GUI Address")
 	}
 
 	if guiCfg.APIKey == "" {
-		return config.GUIConfiguration{}, errors.New("could not find GUI API key")
+		return config.GUIConfiguration{}, errors.New("Could not find GUI API key")
 	}
 
 	return guiCfg, nil
@@ -114,7 +113,7 @@ func (c *apiClient) Endpoint() string {
 }
 
 func (c *apiClient) Do(req *http.Request) (*http.Response, error) {
-	req.Header.Set("X-Api-Key", c.apikey)
+	req.Header.Set("X-API-Key", c.apikey)
 	resp, err := c.Client.Do(req)
 	if err != nil {
 		return nil, err

cmd/syncthing/cli/config.go

@@ -10,7 +10,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"net/http"
 	"reflect"
 
 	"github.com/AudriusButkevicius/recli"
@@ -19,53 +18,6 @@ import (
 	"github.com/urfave/cli"
 )
 
-// Try to mimic the kong output format through custom help templates
-var customAppHelpTemplate = `Usage: {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}}{{if .Commands}} <command> [flags]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}
-
-{{.Description}}{{if .VisibleFlags}}
-
-Flags:
-  {{range $index, $option := .VisibleFlags}}{{if $index}}
-  {{end}}{{$option}}{{end}}{{end}}{{if .VisibleCommands}}
-
-Commands:{{range .VisibleCategories}}{{if .Name}}
-
-  {{.Name}}:{{range .VisibleCommands}}
-    {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}{{else}}{{range .VisibleCommands}}
-  {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}{{end}}{{end}}{{end}}
-`
-
-var customCommandHelpTemplate = `Usage: {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}}{{if .VisibleFlags}} [flags]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}
-
-{{.Usage}}{{if .VisibleFlags}}
-
-Flags:
-  {{range $index, $option := .VisibleFlags}}{{if $index}}
-  {{end}}{{$option}}{{end}}{{end}}{{if .Category}}
-
-Category:
-  {{.Category}}{{end}}{{if .Description}}
-
-{{.Description}}{{end}}
-`
-
-var customSubcommandHelpTemplate = `Usage: {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} <command>{{if .VisibleFlags}} [flags]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}{{if .Description}}
-
-{{.Description}}{{else}}{{if .Usage}}
-
-{{.Usage}}{{end}}{{end}}{{if .VisibleFlags}}
-
-Flags:
-  {{range $index, $option := .VisibleFlags}}{{if $index}}
-  {{end}}{{$option}}{{end}}{{end}}{{if .VisibleCommands}}
-
-Commands:{{range .VisibleCategories}}{{if .Name}}
-
-  {{.Name}}:{{range .VisibleCommands}}
-    {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}{{else}}{{range .VisibleCommands}}
-  {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}{{end}}{{end}}{{end}}
-`
-
 type configHandler struct {
 	original, cfg config.Configuration
 	client        APIClient
@@ -76,18 +28,13 @@ type configCommand struct {
 	Args []string `arg:"" default:"-h"`
 }
 
-func (c *configCommand) Run(ctx Context, outerCtx *kong.Context) error {
+func (c *configCommand) Run(ctx Context, _ *kong.Context) error {
 	app := cli.NewApp()
-	app.Name = "syncthing cli config"
-	app.HelpName = "syncthing cli config"
-	app.Description = outerCtx.Selected().Help
+	app.Name = "syncthing"
+	app.Author = "The Syncthing Authors"
 	app.Metadata = map[string]interface{}{
 		"clientFactory": ctx.clientFactory,
 	}
-	app.CustomAppHelpTemplate = customAppHelpTemplate
-	// Override global templates, as this is out only usage of the package
-	cli.CommandHelpTemplate = customCommandHelpTemplate
-	cli.SubcommandHelpTemplate = customSubcommandHelpTemplate
 
 	h := new(configHandler)
 	h.client, h.err = ctx.clientFactory.getClient()
@@ -108,8 +55,6 @@ func (c *configCommand) Run(ctx Context, outerCtx *kong.Context) error {
 
 	app.Commands = commands
 	app.HideHelp = true
-	// Explicitly re-add help only as flags, not as commands
-	app.Flags = []cli.Flag{cli.HelpFlag}
 	app.Before = h.configBefore
 	app.After = h.configAfter
 
@@ -141,7 +86,7 @@ func (h *configHandler) configAfter(_ *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	if resp.StatusCode != http.StatusOK {
+	if resp.StatusCode != 200 {
 		body, err := responseToBArray(resp)
 		if err != nil {
 			return err

cmd/syncthing/cli/debug.go

@@ -41,4 +41,5 @@ func (p *profileCommand) Run(ctx Context) error {
 type debugCommand struct {
 	File    fileCommand    `cmd:"" help:"Show information about a file (or directory/symlink)"`
 	Profile profileCommand `cmd:"" help:"Save a profile to help figuring out what Syncthing does"`
+	Index   indexCommand   `cmd:"" help:"Show information about the index (database)"`
 }

cmd/syncthing/cli/errors.go

@@ -9,7 +9,6 @@ package cli
 import (
 	"errors"
 	"fmt"
-	"net/http"
 	"strings"
 
 	"github.com/alecthomas/kong"
@@ -35,7 +34,7 @@ func (e *errorsPushCommand) Run(ctx Context) error {
 	if err != nil {
 		return err
 	}
-	if response.StatusCode != http.StatusOK {
+	if response.StatusCode != 200 {
 		errStr = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
 		bytes, err := responseToBArray(response)
 		if err != nil {

cmd/syncthing/cli/index.go

@@ -0,0 +1,32 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"github.com/alecthomas/kong"
+)
+
+type indexCommand struct {
+	Dump     struct{} `cmd:"" help:"Print the entire db"`
+	DumpSize struct{} `cmd:"" help:"Print the db size of different categories of information"`
+	Check    struct{} `cmd:"" help:"Check the database for inconsistencies"`
+	Account  struct{} `cmd:"" help:"Print key and value size statistics per key type"`
+}
+
+func (*indexCommand) Run(kongCtx *kong.Context) error {
+	switch kongCtx.Selected().Name {
+	case "dump":
+		return indexDump()
+	case "dump-size":
+		return indexDumpSize()
+	case "check":
+		return indexCheck()
+	case "account":
+		return indexAccount()
+	}
+	return nil
+}

cmd/syncthing/cli/index_accounting.go

@@ -0,0 +1,62 @@
+// Copyright (C) 2020 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"fmt"
+	"os"
+	"text/tabwriter"
+)
+
+// indexAccount prints key and data size statistics per class
+func indexAccount() error {
+	ldb, err := getDB()
+	if err != nil {
+		return err
+	}
+
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		return err
+	}
+
+	var ksizes [256]int
+	var dsizes [256]int
+	var counts [256]int
+	var max [256]int
+
+	for it.Next() {
+		key := it.Key()
+		t := key[0]
+		ds := len(it.Value())
+		ks := len(key)
+		s := ks + ds
+
+		counts[t]++
+		ksizes[t] += ks
+		dsizes[t] += ds
+		if s > max[t] {
+			max[t] = s
+		}
+	}
+
+	tw := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', tabwriter.AlignRight)
+	toti, totds, totks := 0, 0, 0
+	for t := range ksizes {
+		if ksizes[t] > 0 {
+			// yes metric kilobytes 🤘
+			fmt.Fprintf(tw, "0x%02x:\t%d items,\t%d KB keys +\t%d KB data,\t%d B +\t%d B avg,\t%d B max\t\n", t, counts[t], ksizes[t]/1000, dsizes[t]/1000, ksizes[t]/counts[t], dsizes[t]/counts[t], max[t])
+			toti += counts[t]
+			totds += dsizes[t]
+			totks += ksizes[t]
+		}
+	}
+	fmt.Fprintf(tw, "Total\t%d items,\t%d KB keys +\t%d KB data.\t\n", toti, totks/1000, totds/1000)
+	tw.Flush()
+
+	return nil
+}

cmd/syncthing/cli/index_dump.go

@@ -0,0 +1,158 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"encoding/binary"
+	"fmt"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func indexDump() error {
+	ldb, err := getDB()
+	if err != nil {
+		return err
+	}
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		return err
+	}
+	for it.Next() {
+		key := it.Key()
+		switch key[0] {
+		case db.KeyTypeDevice:
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+			fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
+
+			var f protocol.FileInfo
+			err := f.Unmarshal(it.Value())
+			if err != nil {
+				return err
+			}
+			fmt.Printf(" V:%v\n", f)
+
+		case db.KeyTypeGlobal:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			var flv db.VersionList
+			flv.Unmarshal(it.Value())
+			fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
+
+		case db.KeyTypeBlock:
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[1+4 : 1+4+32]
+			name := nulString(key[1+4+32:])
+			fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
+
+		case db.KeyTypeDeviceStatistic:
+			fmt.Printf("[dstat] K:%x V:%x\n", key, it.Value())
+
+		case db.KeyTypeFolderStatistic:
+			fmt.Printf("[fstat] K:%x V:%x\n", key, it.Value())
+
+		case db.KeyTypeVirtualMtime:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			val := it.Value()
+			var realTime, virtualTime time.Time
+			realTime.UnmarshalBinary(val[:len(val)/2])
+			virtualTime.UnmarshalBinary(val[len(val)/2:])
+			fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, realTime, virtualTime)
+
+		case db.KeyTypeFolderIdx:
+			key := binary.BigEndian.Uint32(key[1:])
+			fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
+
+		case db.KeyTypeDeviceIdx:
+			key := binary.BigEndian.Uint32(key[1:])
+			val := it.Value()
+			device := "<nil>"
+			if len(val) > 0 {
+				dev, err := protocol.DeviceIDFromBytes(val)
+				if err != nil {
+					device = fmt.Sprintf("<invalid %d bytes>", len(val))
+				} else {
+					device = dev.String()
+				}
+			}
+			fmt.Printf("[deviceidx] K:%d V:%s\n", key, device)
+
+		case db.KeyTypeIndexID:
+			device := binary.BigEndian.Uint32(key[1:])
+			folder := binary.BigEndian.Uint32(key[5:])
+			fmt.Printf("[indexid] D:%d F:%d I:%x\n", device, folder, it.Value())
+
+		case db.KeyTypeFolderMeta:
+			folder := binary.BigEndian.Uint32(key[1:])
+			fmt.Printf("[foldermeta] F:%d", folder)
+			var cs db.CountsSet
+			if err := cs.Unmarshal(it.Value()); err != nil {
+				fmt.Printf(" (invalid)\n")
+			} else {
+				fmt.Printf(" V:%v\n", cs)
+			}
+
+		case db.KeyTypeMiscData:
+			fmt.Printf("[miscdata] K:%q V:%q\n", key[1:], it.Value())
+
+		case db.KeyTypeSequence:
+			folder := binary.BigEndian.Uint32(key[1:])
+			seq := binary.BigEndian.Uint64(key[5:])
+			fmt.Printf("[sequence] F:%d S:%d V:%q\n", folder, seq, it.Value())
+
+		case db.KeyTypeNeed:
+			folder := binary.BigEndian.Uint32(key[1:])
+			file := string(key[5:])
+			fmt.Printf("[need] F:%d V:%q\n", folder, file)
+
+		case db.KeyTypeBlockList:
+			fmt.Printf("[blocklist] H:%x\n", key[1:])
+
+		case db.KeyTypeBlockListMap:
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[5:37]
+			fileName := string(key[37:])
+			fmt.Printf("[blocklistmap] F:%d H:%x N:%s\n", folder, hash, fileName)
+
+		case db.KeyTypeVersion:
+			fmt.Printf("[version] H:%x", key[1:])
+			var v protocol.Vector
+			err := v.Unmarshal(it.Value())
+			if err != nil {
+				fmt.Printf(" (invalid)\n")
+			} else {
+				fmt.Printf(" V:%v\n", v)
+			}
+
+		case db.KeyTypePendingFolder:
+			device := binary.BigEndian.Uint32(key[1:])
+			folder := string(key[5:])
+			var of db.ObservedFolder
+			of.Unmarshal(it.Value())
+			fmt.Printf("[pendingFolder] D:%d F:%s V:%v\n", device, folder, of)
+
+		case db.KeyTypePendingDevice:
+			device := "<invalid>"
+			dev, err := protocol.DeviceIDFromBytes(key[1:])
+			if err == nil {
+				device = dev.String()
+			}
+			var od db.ObservedDevice
+			od.Unmarshal(it.Value())
+			fmt.Printf("[pendingDevice] D:%v V:%v\n", device, od)
+
+		default:
+			fmt.Printf("[??? %d]\n  %x\n  %x\n", key[0], key, it.Value())
+		}
+	}
+	return nil
+}

cmd/syncthing/cli/index_dumpsize.go

@@ -0,0 +1,88 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"encoding/binary"
+	"fmt"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/db"
+)
+
+func indexDumpSize() error {
+	type sizedElement struct {
+		key  string
+		size int
+	}
+
+	ldb, err := getDB()
+	if err != nil {
+		return err
+	}
+
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		return err
+	}
+
+	var elems []sizedElement
+	for it.Next() {
+		var ele sizedElement
+
+		key := it.Key()
+		switch key[0] {
+		case db.KeyTypeDevice:
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+			ele.key = fmt.Sprintf("DEVICE:%d:%d:%s", folder, device, name)
+
+		case db.KeyTypeGlobal:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			ele.key = fmt.Sprintf("GLOBAL:%d:%s", folder, name)
+
+		case db.KeyTypeBlock:
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[1+4 : 1+4+32]
+			name := nulString(key[1+4+32:])
+			ele.key = fmt.Sprintf("BLOCK:%d:%x:%s", folder, hash, name)
+
+		case db.KeyTypeDeviceStatistic:
+			ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])
+
+		case db.KeyTypeFolderStatistic:
+			ele.key = fmt.Sprintf("FOLDERSTATS:%s", key[1:])
+
+		case db.KeyTypeVirtualMtime:
+			ele.key = fmt.Sprintf("MTIME:%s", key[1:])
+
+		case db.KeyTypeFolderIdx:
+			id := binary.BigEndian.Uint32(key[1:])
+			ele.key = fmt.Sprintf("FOLDERIDX:%d", id)
+
+		case db.KeyTypeDeviceIdx:
+			id := binary.BigEndian.Uint32(key[1:])
+			ele.key = fmt.Sprintf("DEVICEIDX:%d", id)
+
+		default:
+			ele.key = fmt.Sprintf("UNKNOWN:%x", key)
+		}
+		ele.size = len(it.Value())
+		elems = append(elems, ele)
+	}
+
+	sort.Slice(elems, func(i, j int) bool {
+		return elems[i].size > elems[j].size
+	})
+	for _, ele := range elems {
+		fmt.Println(ele.key, ele.size)
+	}
+
+	return nil
+}

cmd/syncthing/cli/index_idxck.go

@@ -0,0 +1,355 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+type fileInfoKey struct {
+	folder uint32
+	device uint32
+	name   string
+}
+
+type globalKey struct {
+	folder uint32
+	name   string
+}
+
+type sequenceKey struct {
+	folder   uint32
+	sequence uint64
+}
+
+func indexCheck() (err error) {
+	ldb, err := getDB()
+	if err != nil {
+		return err
+	}
+
+	folders := make(map[uint32]string)
+	devices := make(map[uint32]string)
+	deviceToIDs := make(map[string]uint32)
+	fileInfos := make(map[fileInfoKey]protocol.FileInfo)
+	globals := make(map[globalKey]db.VersionList)
+	sequences := make(map[sequenceKey]string)
+	needs := make(map[globalKey]struct{})
+	blocklists := make(map[string]struct{})
+	versions := make(map[string]protocol.Vector)
+	usedBlocklists := make(map[string]struct{})
+	usedVersions := make(map[string]struct{})
+	var localDeviceKey uint32
+	success := true
+	defer func() {
+		if err == nil {
+			if success {
+				fmt.Println("Index check completed successfully.")
+			} else {
+				err = errors.New("Inconsistencies found in the index")
+			}
+		}
+	}()
+
+	it, err := ldb.NewPrefixIterator(nil)
+	if err != nil {
+		return err
+	}
+	for it.Next() {
+		key := it.Key()
+		switch key[0] {
+		case db.KeyTypeDevice:
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+
+			var f protocol.FileInfo
+			err := f.Unmarshal(it.Value())
+			if err != nil {
+				fmt.Println("Unable to unmarshal FileInfo:", err)
+				success = false
+				continue
+			}
+
+			fileInfos[fileInfoKey{folder, device, name}] = f
+
+		case db.KeyTypeGlobal:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			var flv db.VersionList
+			if err := flv.Unmarshal(it.Value()); err != nil {
+				fmt.Println("Unable to unmarshal VersionList:", err)
+				success = false
+				continue
+			}
+			globals[globalKey{folder, name}] = flv
+
+		case db.KeyTypeFolderIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			folders[key] = string(it.Value())
+
+		case db.KeyTypeDeviceIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			devices[key] = string(it.Value())
+			deviceToIDs[string(it.Value())] = key
+			if bytes.Equal(it.Value(), protocol.LocalDeviceID[:]) {
+				localDeviceKey = key
+			}
+
+		case db.KeyTypeSequence:
+			folder := binary.BigEndian.Uint32(key[1:])
+			seq := binary.BigEndian.Uint64(key[5:])
+			val := it.Value()
+			sequences[sequenceKey{folder, seq}] = string(val[9:])
+
+		case db.KeyTypeNeed:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			needs[globalKey{folder, name}] = struct{}{}
+
+		case db.KeyTypeBlockList:
+			hash := string(key[1:])
+			blocklists[hash] = struct{}{}
+
+		case db.KeyTypeVersion:
+			hash := string(key[1:])
+			var v protocol.Vector
+			if err := v.Unmarshal(it.Value()); err != nil {
+				fmt.Println("Unable to unmarshal Vector:", err)
+				success = false
+				continue
+			}
+			versions[hash] = v
+		}
+	}
+
+	if localDeviceKey == 0 {
+		fmt.Println("Missing key for local device in device index (bailing out)")
+		success = false
+		return
+	}
+
+	var missingSeq []sequenceKey
+	for fk, fi := range fileInfos {
+		if fk.name != fi.Name {
+			fmt.Printf("Mismatching FileInfo name, %q (key) != %q (actual)\n", fk.name, fi.Name)
+			success = false
+		}
+
+		folder := folders[fk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for FileInfo %q\n", fk.folder, fk.name)
+			success = false
+			continue
+		}
+		if devices[fk.device] == "" {
+			fmt.Printf("Unknown device ID %d for FileInfo %q, folder %q\n", fk.folder, fk.name, folder)
+			success = false
+		}
+
+		if fk.device == localDeviceKey {
+			sk := sequenceKey{fk.folder, uint64(fi.Sequence)}
+			name, ok := sequences[sk]
+			if !ok {
+				fmt.Printf("Sequence entry missing for FileInfo %q, folder %q, seq %d\n", fi.Name, folder, fi.Sequence)
+				missingSeq = append(missingSeq, sk)
+				success = false
+				continue
+			}
+			if name != fi.Name {
+				fmt.Printf("Sequence entry refers to wrong name, %q (seq) != %q (FileInfo), folder %q, seq %d\n", name, fi.Name, folder, fi.Sequence)
+				success = false
+			}
+		}
+
+		if len(fi.Blocks) == 0 && len(fi.BlocksHash) != 0 {
+			key := string(fi.BlocksHash)
+			if _, ok := blocklists[key]; !ok {
+				fmt.Printf("Missing block list for file %q, block list hash %x\n", fi.Name, fi.BlocksHash)
+				success = false
+			} else {
+				usedBlocklists[key] = struct{}{}
+			}
+		}
+
+		if fi.VersionHash != nil {
+			key := string(fi.VersionHash)
+			if _, ok := versions[key]; !ok {
+				fmt.Printf("Missing version vector for file %q, version hash %x\n", fi.Name, fi.VersionHash)
+				success = false
+			} else {
+				usedVersions[key] = struct{}{}
+			}
+		}
+
+		_, ok := globals[globalKey{fk.folder, fk.name}]
+		if !ok {
+			fmt.Printf("Missing global for file %q\n", fi.Name)
+			success = false
+			continue
+		}
+	}
+
+	// Aggregate the ranges of missing sequence entries, print them
+
+	sort.Slice(missingSeq, func(a, b int) bool {
+		if missingSeq[a].folder != missingSeq[b].folder {
+			return missingSeq[a].folder < missingSeq[b].folder
+		}
+		return missingSeq[a].sequence < missingSeq[b].sequence
+	})
+
+	var folder uint32
+	var startSeq, prevSeq uint64
+	for _, sk := range missingSeq {
+		if folder != sk.folder || sk.sequence != prevSeq+1 {
+			if folder != 0 {
+				fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
+			}
+			startSeq = sk.sequence
+			folder = sk.folder
+		}
+		prevSeq = sk.sequence
+	}
+	if folder != 0 {
+		fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
+	}
+
+	for gk, vl := range globals {
+		folder := folders[gk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
+			success = false
+		}
+		checkGlobal := func(i int, device []byte, version protocol.Vector, invalid, deleted bool) {
+			dev, ok := deviceToIDs[string(device)]
+			if !ok {
+				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, device)
+				success = false
+			}
+			fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
+			if !ok {
+				fmt.Printf("VersionList %q, folder %q, entry %d refers to unknown FileInfo\n", gk.name, folder, i)
+				success = false
+			}
+
+			fiv := fi.Version
+			if fi.VersionHash != nil {
+				fiv = versions[string(fi.VersionHash)]
+			}
+			if !fiv.Equal(version) {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, version, fi.Version)
+				success = false
+			}
+			if fi.IsInvalid() != invalid {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, invalid, fi.IsInvalid())
+				success = false
+			}
+			if fi.IsDeleted() != deleted {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo deleted mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, deleted, fi.IsDeleted())
+				success = false
+			}
+		}
+		for i, fv := range vl.RawVersions {
+			for _, device := range fv.Devices {
+				checkGlobal(i, device, fv.Version, false, fv.Deleted)
+			}
+			for _, device := range fv.InvalidDevices {
+				checkGlobal(i, device, fv.Version, true, fv.Deleted)
+			}
+		}
+
+		// If we need this file we should have a need entry for it. False
+		// positives from needsLocally for deleted files, where we might
+		// legitimately lack an entry if we never had it, and ignored files.
+		if needsLocally(vl) {
+			_, ok := needs[gk]
+			if !ok {
+				fv, _ := vl.GetGlobal()
+				devB, _ := fv.FirstDevice()
+				dev := deviceToIDs[string(devB)]
+				fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
+				if !fi.IsDeleted() && !fi.IsIgnored() {
+					fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
+				}
+			}
+		}
+	}
+
+	seenSeq := make(map[fileInfoKey]uint64)
+	for sk, name := range sequences {
+		folder := folders[sk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for sequence entry %d, %q\n", sk.folder, sk.sequence, name)
+			success = false
+			continue
+		}
+
+		if prev, ok := seenSeq[fileInfoKey{folder: sk.folder, name: name}]; ok {
+			fmt.Printf("Duplicate sequence entry for %q, folder %q, seq %d (prev %d)\n", name, folder, sk.sequence, prev)
+			success = false
+		}
+		seenSeq[fileInfoKey{folder: sk.folder, name: name}] = sk.sequence
+
+		fi, ok := fileInfos[fileInfoKey{sk.folder, localDeviceKey, name}]
+		if !ok {
+			fmt.Printf("Missing FileInfo for sequence entry %d, folder %q, %q\n", sk.sequence, folder, name)
+			success = false
+			continue
+		}
+		if fi.Sequence != int64(sk.sequence) {
+			fmt.Printf("Sequence mismatch for %q, folder %q, %d (key) != %d (FileInfo)\n", name, folder, sk.sequence, fi.Sequence)
+			success = false
+		}
+	}
+
+	for nk := range needs {
+		folder := folders[nk.folder]
+		if folder == "" {
+			fmt.Printf("Unknown folder ID %d for need entry %q\n", nk.folder, nk.name)
+			success = false
+			continue
+		}
+
+		vl, ok := globals[nk]
+		if !ok {
+			fmt.Printf("Missing global for need entry %q, folder %q\n", nk.name, folder)
+			success = false
+			continue
+		}
+
+		if !needsLocally(vl) {
+			fmt.Printf("Need entry for file we don't need, %q, folder %q\n", nk.name, folder)
+			success = false
+		}
+	}
+
+	if d := len(blocklists) - len(usedBlocklists); d > 0 {
+		fmt.Printf("%d block list entries out of %d needs GC\n", d, len(blocklists))
+	}
+	if d := len(versions) - len(usedVersions); d > 0 {
+		fmt.Printf("%d version entries out of %d needs GC\n", d, len(versions))
+	}
+
+	return nil
+}
+
+func needsLocally(vl db.VersionList) bool {
+	gfv, gok := vl.GetGlobal()
+	if !gok { // That's weird, but we hardly need something non-existent
+		return false
+	}
+	fv, ok := vl.Get(protocol.LocalDeviceID[:])
+	return db.Need(gfv, ok, fv.Version)
+}

cmd/syncthing/cli/main.go

@@ -12,14 +12,17 @@ import (
 	"os"
 
 	"github.com/alecthomas/kong"
-	"github.com/kballard/go-shellquote"
+	"github.com/flynn-archive/go-shlex"
 
+	"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
 	"github.com/syncthing/syncthing/lib/config"
 )
 
 type CLI struct {
-	GUIAddress string `name:"gui-address" env:"STGUIADDRESS"`
-	GUIAPIKey  string `name:"gui-apikey" env:"STGUIAPIKEY"`
+	cmdutil.CommonOptions
+	DataDir    string `name:"data" placeholder:"PATH" env:"STDATADIR" help:"Set data directory (database and logs)"`
+	GUIAddress string `name:"gui-address"`
+	GUIAPIKey  string `name:"gui-apikey"`
 
 	Show       showCommand      `cmd:"" help:"Show command group"`
 	Debug      debugCommand     `cmd:"" help:"Debug command group"`
@@ -34,6 +37,11 @@ type Context struct {
 }
 
 func (cli CLI) AfterApply(kongCtx *kong.Context) error {
+	err := cmdutil.SetConfigDataLocationsFromFlags(cli.HomeDir, cli.ConfDir, cli.DataDir)
+	if err != nil {
+		return fmt.Errorf("command line options: %w", err)
+	}
+
 	clientFactory := &apiClientFactory{
 		cfg: config.GUIConfiguration{
 			RawAddress: cli.GUIAddress,
@@ -59,7 +67,7 @@ func (*stdinCommand) Run() error {
 	fmt.Println("Reading commands from stdin...", args)
 	scanner := bufio.NewScanner(os.Stdin)
 	for scanner.Scan() {
-		input, err := shellquote.Split(scanner.Text())
+		input, err := shlex.Split(scanner.Text())
 		if err != nil {
 			return fmt.Errorf("parsing input: %w", err)
 		}

cmd/syncthing/cli/operations.go

@@ -10,7 +10,6 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
-	"net/http"
 	"path/filepath"
 
 	"github.com/alecthomas/kong"
@@ -64,7 +63,7 @@ func (f *folderOverrideCommand) Run(ctx Context) error {
 			if err != nil {
 				return err
 			}
-			if response.StatusCode != http.StatusOK {
+			if response.StatusCode != 200 {
 				errStr := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
 				bytes, err := responseToBArray(response)
 				if err != nil {

cmd/syncthing/cli/utils.go

@@ -17,6 +17,8 @@ import (
 	"path/filepath"
 
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/db/backend"
+	"github.com/syncthing/syncthing/lib/locations"
 )
 
 func responseToBArray(response *http.Response) ([]byte, error) {
@@ -131,6 +133,19 @@ func prettyPrintResponse(response *http.Response) error {
 	return prettyPrintJSON(data)
 }
 
+func getDB() (backend.Backend, error) {
+	return backend.OpenLevelDBRO(locations.Get(locations.Database))
+}
+
+func nulString(bs []byte) string {
+	for i := range bs {
+		if bs[i] == 0 {
+			return string(bs[:i])
+		}
+	}
+	return string(bs)
+}
+
 func normalizePath(path string) string {
 	return filepath.ToSlash(filepath.Clean(path))
 }

cmd/syncthing/cmdutil/options_common.go

@@ -0,0 +1,16 @@
+// Copyright (C) 2021 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cmdutil
+
+// CommonOptions are reused among several subcommands
+type CommonOptions struct {
+	buildCommonOptions
+	ConfDir         string `name:"config" placeholder:"PATH" env:"STCONFDIR" help:"Set configuration directory (config and keys)"`
+	HomeDir         string `name:"home" placeholder:"PATH" env:"STHOMEDIR" help:"Set configuration and data directory"`
+	NoDefaultFolder bool   `env:"STNODEFAULTFOLDER" help:"Don't create the \"default\" folder on first startup"`
+	SkipPortProbing bool   `help:"Don't try to find free ports for GUI and listen addresses on first startup"`
+}

cmd/syncthing/cmdutil/options_others.go

@@ -7,8 +7,8 @@
 //go:build !windows
 // +build !windows
 
-package main
+package cmdutil
 
-type buildSpecificOptions struct {
+type buildCommonOptions struct {
 	HideConsole bool `hidden:""`
 }

cmd/syncthing/cmdutil/options_windows.go

@@ -4,8 +4,8 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package main
+package cmdutil
 
-type buildSpecificOptions struct {
-	HideConsole bool `name:"no-console" help:"Hide console window" env:"STHIDECONSOLE"`
+type buildCommonOptions struct {
+	HideConsole bool `name:"no-console" help:"Hide console window"`
 }

cmd/syncthing/cmdutil/util.go

@@ -0,0 +1,35 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cmdutil
+
+import (
+	"errors"
+
+	"github.com/syncthing/syncthing/lib/locations"
+)
+
+func SetConfigDataLocationsFromFlags(homeDir, confDir, dataDir string) error {
+	homeSet := homeDir != ""
+	confSet := confDir != ""
+	dataSet := dataDir != ""
+	switch {
+	case dataSet != confSet:
+		return errors.New("either both or none of --config and --data must be given, use --home to set both at once")
+	case homeSet && dataSet:
+		return errors.New("--home must not be used together with --config and --data")
+	case homeSet:
+		confDir = homeDir
+		dataDir = homeDir
+		fallthrough
+	case dataSet:
+		if err := locations.SetBaseDir(locations.ConfigBaseDir, confDir); err != nil {
+			return err
+		}
+		return locations.SetBaseDir(locations.DataBaseDir, dataDir)
+	}
+	return nil
+}

cmd/syncthing/crash_reporting.go

@@ -9,17 +9,15 @@ package main
 import (
 	"bytes"
 	"context"
-	"crypto/sha256"
 	"fmt"
-	"log/slog"
 	"net/http"
 	"os"
 	"path/filepath"
-	"slices"
+	"sort"
 	"strings"
 	"time"
 
-	"github.com/syncthing/syncthing/internal/slogutil"
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 const (
@@ -31,18 +29,16 @@ const (
 // directory to the crash reporting server as urlBase. Uploads are attempted
 // with the newest log first.
 //
-// This can block for a long time. The context can set a final deadline
+// This can can block for a long time. The context can set a final deadline
 // for this.
 func uploadPanicLogs(ctx context.Context, urlBase, dir string) {
 	files, err := filepath.Glob(filepath.Join(dir, "panic-*.log"))
 	if err != nil {
-		slog.ErrorContext(ctx, "Failed to list panic logs", slogutil.Error(err))
+		l.Warnln("Failed to list panic logs:", err)
 		return
 	}
 
-	slices.SortFunc(files, func(a, b string) int {
-		return strings.Compare(b, a)
-	})
+	sort.Sort(sort.Reverse(sort.StringSlice(files)))
 	for _, file := range files {
 		if strings.Contains(file, ".reported.") {
 			// We've already sent this file. It'll be cleaned out at some
@@ -51,7 +47,7 @@ func uploadPanicLogs(ctx context.Context, urlBase, dir string) {
 		}
 
 		if err := uploadPanicLog(ctx, urlBase, file); err != nil {
-			slog.ErrorContext(ctx, "Reporting crash", slogutil.Error(err))
+			l.Warnln("Reporting crash:", err)
 		} else {
 			// Rename the log so we don't have to try to report it again. This
 			// succeeds, or it does not. There is no point complaining about it.
@@ -74,7 +70,7 @@ func uploadPanicLog(ctx context.Context, urlBase, file string) error {
 	data = filterLogLines(data)
 
 	hash := fmt.Sprintf("%x", sha256.Sum256(data))
-	slog.InfoContext(ctx, "Reporting crash", slogutil.FilePath(filepath.Base(file)), slog.String("id", hash[:8]))
+	l.Infof("Reporting crash found in %s (report ID %s) ...\n", filepath.Base(file), hash[:8])
 
 	url := fmt.Sprintf("%s/%s", urlBase, hash)
 	headReq, err := http.NewRequest(http.MethodHead, url, nil)

cmd/syncthing/debug.go

@@ -6,6 +6,10 @@
 
 package main
 
-import "github.com/syncthing/syncthing/internal/slogutil"
+import (
+	"github.com/syncthing/syncthing/lib/logger"
+)
 
-func init() { slogutil.RegisterPackage("Main package") }
+var (
+	l = logger.DefaultLogger.NewFacility("main", "Main package")
+)

cmd/syncthing/decrypt/decrypt.go

@@ -17,9 +17,6 @@ import (
 	"os"
 	"path/filepath"
 
-	"google.golang.org/protobuf/proto"
-
-	"github.com/syncthing/syncthing/internal/gen/bep"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/osutil"
@@ -238,7 +235,7 @@ func (c *CLI) decryptFile(encFi *protocol.FileInfo, plainFi *protocol.FileInfo,
 		}
 
 		// Verify the hash against the plaintext block info
-		if !scanner.Validate(dec, plainBlock.Hash) {
+		if !scanner.Validate(dec, plainBlock.Hash, 0) {
 			// The block decrypted correctly but fails the hash check. This
 			// is odd and unexpected, but it it's still a valid block from
 			// the source. The file might have changed while we pulled it?
@@ -283,11 +280,10 @@ func loadEncryptedFileInfo(fd fs.File) (*protocol.FileInfo, error) {
 		return nil, err
 	}
 
-	var encFi bep.FileInfo
-	if err := proto.Unmarshal(trailer, &encFi); err != nil {
+	var encFi protocol.FileInfo
+	if err := encFi.Unmarshal(trailer); err != nil {
 		return nil, err
 	}
-	fi := protocol.FileInfoFromWire(&encFi)
 
-	return &fi, nil
+	return &encFi, nil
 }

cmd/syncthing/generate/generate.go

@@ -11,25 +11,42 @@ import (
 	"bufio"
 	"context"
 	"crypto/tls"
+	"errors"
 	"fmt"
-	"log/slog"
 	"os"
 
+	"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/locations"
+	"github.com/syncthing/syncthing/lib/logger"
+	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/syncthing"
 )
 
 type CLI struct {
-	GUIUser       string `placeholder:"STRING" help:"Specify new GUI authentication user name"`
-	GUIPassword   string `placeholder:"STRING" help:"Specify new GUI authentication password (use - to read from standard input)"`
-	NoPortProbing bool   `help:"Don't try to find free ports for GUI and listen addresses on first startup" env:"STNOPORTPROBING"`
+	cmdutil.CommonOptions
+	GUIUser     string `placeholder:"STRING" help:"Specify new GUI authentication user name"`
+	GUIPassword string `placeholder:"STRING" help:"Specify new GUI authentication password (use - to read from standard input)"`
 }
 
-func (c *CLI) Run() error {
+func (c *CLI) Run(l logger.Logger) error {
+	if c.HideConsole {
+		osutil.HideConsole()
+	}
+
+	if c.HomeDir != "" {
+		if c.ConfDir != "" {
+			return errors.New("--home must not be used together with --config")
+		}
+		c.ConfDir = c.HomeDir
+	}
+	if c.ConfDir == "" {
+		c.ConfDir = locations.GetBaseDir(locations.ConfigBaseDir)
+	}
+
 	// Support reading the password from a pipe or similar
 	if c.GUIPassword == "-" {
 		reader := bufio.NewReader(os.Stdin)
@@ -40,13 +57,13 @@ func (c *CLI) Run() error {
 		c.GUIPassword = string(password)
 	}
 
-	if err := Generate(locations.GetBaseDir(locations.ConfigBaseDir), c.GUIUser, c.GUIPassword, c.NoPortProbing); err != nil {
+	if err := Generate(l, c.ConfDir, c.GUIUser, c.GUIPassword, c.NoDefaultFolder, c.SkipPortProbing); err != nil {
 		return fmt.Errorf("failed to generate config and keys: %w", err)
 	}
 	return nil
 }
 
-func Generate(confDir, guiUser, guiPassword string, skipPortProbing bool) error {
+func Generate(l logger.Logger, confDir, guiUser, guiPassword string, noDefaultFolder, skipPortProbing bool) error {
 	dir, err := fs.ExpandTilde(confDir)
 	if err != nil {
 		return err
@@ -61,21 +78,20 @@ func Generate(confDir, guiUser, guiPassword string, skipPortProbing bool) error
 	certFile, keyFile := locations.Get(locations.CertFile), locations.Get(locations.KeyFile)
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err == nil {
-		slog.Warn("Key exists; will not overwrite")
+		l.Warnln("Key exists; will not overwrite.")
 	} else {
 		cert, err = syncthing.GenerateCertificate(certFile, keyFile)
 		if err != nil {
 			return fmt.Errorf("create certificate: %w", err)
 		}
 	}
-
 	myID = protocol.NewDeviceID(cert.Certificate[0])
-	slog.Info("Calculated device ID", slog.String("device", myID.String()))
+	l.Infoln("Device ID:", myID)
 
 	cfgFile := locations.Get(locations.ConfigFile)
 	cfg, _, err := config.Load(cfgFile, myID, events.NoopLogger)
 	if fs.IsNotExist(err) {
-		if cfg, err = syncthing.DefaultConfig(cfgFile, myID, events.NoopLogger, skipPortProbing); err != nil {
+		if cfg, err = syncthing.DefaultConfig(cfgFile, myID, events.NoopLogger, noDefaultFolder, skipPortProbing); err != nil {
 			return fmt.Errorf("create config: %w", err)
 		}
 	} else if err != nil {
@@ -88,7 +104,7 @@ func Generate(confDir, guiUser, guiPassword string, skipPortProbing bool) error
 
 	var updateErr error
 	waiter, err := cfg.Modify(func(cfg *config.Configuration) {
-		updateErr = updateGUIAuthentication(&cfg.GUI, guiUser, guiPassword)
+		updateErr = updateGUIAuthentication(l, &cfg.GUI, guiUser, guiPassword)
 	})
 	if err != nil {
 		return fmt.Errorf("modify config: %w", err)
@@ -104,17 +120,17 @@ func Generate(confDir, guiUser, guiPassword string, skipPortProbing bool) error
 	return nil
 }
 
-func updateGUIAuthentication(guiCfg *config.GUIConfiguration, guiUser, guiPassword string) error {
+func updateGUIAuthentication(l logger.Logger, guiCfg *config.GUIConfiguration, guiUser, guiPassword string) error {
 	if guiUser != "" && guiCfg.User != guiUser {
 		guiCfg.User = guiUser
-		slog.Info("Updated GUI authentication user", "name", guiUser)
+		l.Infoln("Updated GUI authentication user name:", guiUser)
 	}
 
 	if guiPassword != "" && guiCfg.Password != guiPassword {
 		if err := guiCfg.SetPassword(guiPassword); err != nil {
 			return fmt.Errorf("failed to set GUI authentication password: %w", err)
 		}
-		slog.Info("Updated GUI authentication password")
+		l.Infoln("Updated GUI authentication password.")
 	}
 	return nil
 }