fix(beacon): skip point-to-point interfaces on Android (#10504)

Signed-off-by: Jakob Borg <jakob@kastelo.net>

.github/CODEOWNERS

@@ -1,2 +0,0 @@
-/AUTHORS    @calmh
-/*.md       @calmh

.github/ISSUE_TEMPLATE/01-feature.yml

@@ -1,6 +1,7 @@
 name: Feature request
 description: File a new feature request
 labels: ["enhancement", "needs-triage"]
+type: Feature
 body:
 
   - type: textarea

.github/ISSUE_TEMPLATE/02-bug.yml

@@ -1,6 +1,7 @@
 name: Bug report
 description: If you're actually looking for support instead, see "I need help / I have a question".
 labels: ["bug", "needs-triage"]
+type: Bug
 body:
   - type: markdown
     attributes:

.github/dependabot.yml

@@ -3,11 +3,11 @@ updates:
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: weekly
+      interval: monthly
     open-pull-requests-limit: 10
 
   - package-ecosystem: "gomod"
     directory: "/"
     schedule:
-      interval: weekly
+      interval: monthly
     open-pull-requests-limit: 10

.github/labeler.yml

@@ -0,0 +1,23 @@
+version: 1
+labels:
+
+- label: enhancement
+  title: ^feat\b
+
+- label: bug
+  title: ^fix\b
+
+- label: documentation
+  title: ^docs\b
+
+- label: chore
+  title: ^chore\b
+
+- label: chore
+  title: ^refactor\b
+
+- label: build
+  title: ^build\b
+
+- label: dependencies
+  title: ^build\(deps\)\b

.github/regsync.yml

@@ -0,0 +1,52 @@
+  version: 1
+  creds:
+    - registry: docker.io
+      user: "{{env \"DOCKERHUB_USERNAME\"}}"
+      pass: "{{env \"DOCKERHUB_TOKEN\"}}"
+
+  defaults:
+    ratelimit:
+      min: 100
+      retry: 1m
+    parallel: 4
+
+  sync:
+
+    - source: ghcr.io/syncthing/syncthing
+      target: docker.io/syncthing/syncthing
+      type: repository
+      tags:
+        allow:
+          - latest
+          - rc
+          - edge
+          - \d+
+          - \d+\.\d+
+          - \d+\.\d+\.\d+
+          - \d+\.\d+\.\d+-rc\.\d+
+
+    - source: ghcr.io/syncthing/relaysrv
+      target: docker.io/syncthing/relaysrv
+      type: repository
+      tags:
+        allow:
+          - latest
+          - rc
+          - edge
+          - \d+
+          - \d+\.\d+
+          - \d+\.\d+\.\d+
+          - \d+\.\d+\.\d+-rc\.\d+
+
+    - source: ghcr.io/syncthing/discosrv
+      target: docker.io/syncthing/discosrv
+      type: repository
+      tags:
+        allow:
+          - latest
+          - rc
+          - edge
+          - \d+
+          - \d+\.\d+
+          - \d+\.\d+\.\d+
+          - \d+\.\d+\.\d+-rc\.\d+

.github/release.yml

@@ -0,0 +1,17 @@
+changelog:
+  exclude:
+    labels:
+      - dependencies
+
+  categories:
+    - title: Fixes
+      labels:
+        - bug
+
+    - title: Features
+      labels:
+        - enhancement
+
+    - title: Other
+      labels:
+        - '*'

.github/workflows/build-infra-dockers.yaml

@@ -7,11 +7,15 @@ on:
       - infra-*
 
 env:
-  GO_VERSION: "~1.23.0"
+  GO_VERSION: "~1.25.0"
   CGO_ENABLED: "0"
   BUILD_USER: docker
   BUILD_HOST: github.syncthing.net
 
+permissions:
+  contents: read
+  packages: write
+
 jobs:
   docker-syncthing:
     name: Build and push Docker images
@@ -26,11 +30,11 @@ jobs:
           - stupgrades
           - ursrv
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@v6
         with:
           go-version: ${{ env.GO_VERSION }}
           check-latest: true
@@ -41,6 +45,13 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Build binaries
         run: |
           for arch in arm64 amd64; do
@@ -48,18 +59,21 @@ jobs:
             mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-"$arch"
           done
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
       - name: Set Docker tags (all branches)
         run: |
-          tags=syncthing/${{ matrix.pkg }}:${{ github.sha }}
+          tags=docker.io/syncthing/${{ matrix.pkg }}:${{ github.sha }},ghcr.io/syncthing/infra/${{ matrix.pkg }}:${{ github.sha }}
           echo "TAGS=$tags" >> $GITHUB_ENV
 
       - name: Set Docker tags (latest)
         if: github.ref == 'refs/heads/infrastructure'
         run: |
-          tags=syncthing/${{ matrix.pkg }}:latest,${{ env.TAGS }}
+          tags=docker.io/syncthing/${{ matrix.pkg }}:latest,ghcr.io/syncthing/infra/${{ matrix.pkg }}:latest,${{ env.TAGS }}
           echo "TAGS=$tags" >> $GITHUB_ENV
 
       - name: Build and push

.github/workflows/build-nightly.yaml

@@ -0,0 +1,18 @@
+name: Build Syncthing (Nightly)
+
+on:
+  schedule:
+    # Run nightly build at 05:00 UTC
+    - cron: '00 05 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  packages: write
+
+jobs:
+  build-syncthing:
+    uses: ./.github/workflows/build-syncthing.yaml
+    # if we only want nightlies to run for specific users:
+    # if: contains(fromJSON('["syncthing", "calmh"]'), github.repository_owner)
+    secrets: inherit

.github/workflows/mirrors.yaml

@@ -0,0 +1,18 @@
+name: Mirrors
+
+on: [push, delete]
+
+jobs:
+  codeberg:
+    name: Mirror to Codeberg
+    if: github.repository_owner == 'syncthing'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: yesolutions/mirror-action@master
+        with:
+          REMOTE: ssh://git@codeberg.org/${{ github.repository }}.git
+          GIT_SSH_PRIVATE_KEY: ${{ secrets.CODEBERG_PUSH_KEY }}
+          GIT_SSH_NO_VERIFY_HOST: "true"

.github/workflows/org-members.yaml

@@ -0,0 +1,20 @@
+name: Org membership recommendations
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 1 * *'
+
+jobs:
+
+  run-recommendation:
+    runs-on: ubuntu-latest
+    name: Check for a recommendation
+    steps:
+
+      - uses: docker://ghcr.io/calmh/github-org-members:latest
+        env:
+          GITHUB_ORGANISATION: syncthing
+          GITHUB_TOKEN: ${{ secrets.GOM_GITHUB_TOKEN }}
+          GOM_IGNORE_USERS: ${{ secrets.GOM_IGNORE_USERS }}
+          GOM_ALSO_REPOS: ${{ secrets.GOM_ALSO_REPOS }}

.github/workflows/pr-metadata.yaml

@@ -0,0 +1,27 @@
+name: PR metadata
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - reopened
+      - edited
+      - synchronize
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+
+  #
+  # Set labels on PRs, which are then used to categorise release notes
+  #
+
+  labels:
+    name: Set labels
+    runs-on: ubuntu-latest
+    steps:
+    - uses: srvaroa/labeler@v1
+      env:
+        GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/release-syncthing.yaml

@@ -0,0 +1,60 @@
+name: Release Syncthing
+
+on:
+  push:
+    branches:
+      - release
+      - release-rc*
+
+permissions:
+  contents: write
+
+jobs:
+  create-release-tag:
+    name: Create release tag
+    runs-on: ubuntu-latest
+    environment: release
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+          ref: ${{ github.ref }} # https://github.com/actions/checkout/issues/882
+          token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version: stable
+
+      - name: Determine version to release
+        run: |
+          if [[ "$GITHUB_REF_NAME" == "release" ]] ; then
+            next=$(go run ./script/next-version.go)
+          else
+            next=$(go run ./script/next-version.go --pre)
+          fi
+          echo "NEXT=$next" >> $GITHUB_ENV
+          echo "Next version is $next"
+
+          prev=$(git describe --exclude "*-*" --abbrev=0)
+          echo "PREV=$prev" >> $GITHUB_ENV
+          echo "Previous version is $prev"
+
+      - name: Determine release notes
+        run: |
+          go run ./script/relnotes.go --new-ver "$NEXT" --branch "$GITHUB_REF_NAME" --prev-ver "$PREV" > notes.md
+        env:
+          GITHUB_TOKEN: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
+
+      - name: Create and push tag
+        run: |
+          git config --global user.name 'Syncthing Release Automation'
+          git config --global user.email 'release@syncthing.net'
+          git tag -a -F notes.md --cleanup=whitespace "$NEXT"
+          git push origin "$NEXT"
+
+      - name: Trigger the build
+        uses: benc-uk/workflow-dispatch@v1
+        with:
+          workflow: build-syncthing.yaml
+          ref: refs/tags/${{ env.NEXT }}
+          token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}

.github/workflows/trigger-nightly.yaml

@@ -8,11 +8,12 @@ on:
 jobs:
 
   trigger-nightly:
+    if: github.repository_owner == 'syncthing'
     runs-on: ubuntu-latest
     name: Push to release-nightly to trigger build
     steps:
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
           fetch-depth: 0

.github/workflows/update-docs-translations.yaml

@@ -10,11 +10,11 @@ jobs:
     runs-on: ubuntu-latest
     name: Update translations and documentation
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
           token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@v6
         with:
           go-version: stable
       - run: |

.gitignore

@@ -17,5 +17,4 @@ deb
 *.bz2
 /repos
 /proto/scripts/protoc-gen-gosyncthing
-/gui/next-gen-gui
 /compat.json

.golangci.yml

@@ -1,26 +1,96 @@
-linters-settings:
-  maligned:
-    suggest-new: true
-
+version: "2"
 linters:
-  enable-all: true
+  default: all
   disable:
-    - goimports
+    - cyclop
     - depguard
-    - lll
-    - gochecknoinits
-    - gochecknoglobals
-    - gofmt
-    - scopelint
-    - gocyclo
+    - err113
+    - exhaustive
+    - exhaustruct
+    - forbidigo
+    - funcorder
     - funlen
-    - wsl
+    - gochecknoglobals
+    - gochecknoinits
     - gocognit
+    - goconst
+    - gocyclo
+    - godot
     - godox
-
-service:
-  golangci-lint-version: 1.21.x
-  prepare:
-    - rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
-    - GO111MODULE=on go mod vendor
-    - go run build.go assets
+    - gomoddirectives
+    - inamedparam
+    - interfacebloat
+    - ireturn
+    - lll
+    - maintidx
+    - mnd
+    - musttag
+    - nestif
+    - nlreturn
+    - noinlineerr
+    - nonamedreturns
+    - paralleltest
+    - prealloc
+    - predeclared
+    - protogetter
+    - recvcheck
+    - revive
+    - tagalign
+    - tagliatelle
+    - testpackage
+    - usetesting # go 1.24
+    - varnamelen
+    - whitespace
+    - wrapcheck
+    - wsl
+    - wsl_v5
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    paths:
+      - internal/gen
+      - internal/db/olddb
+      - cmd/dev
+      - repos
+      - third_party$
+      - builtin$
+      - examples$
+      - _test\.go$
+    rules:
+      # relax the slog rules for debug lines, for now
+      - linters: [sloglint]
+        source: Debug
+      # contexts are irrelevant for SQLite
+      - linters: [noctx]
+        text: database/sql
+      # Rollback errors can be ignored
+      - linters: [errcheck]
+        source: Rollback
+      # Embedded fields named in selectors may add clarity
+      - linters: [staticcheck]
+        text: QF1008
+      # Don't necessarily rewrite !(foo || bar) to !foo && !bar
+      - linters: [staticcheck]
+        text: QF1001
+  settings:
+    sloglint:
+      context: "scope"
+      static-msg: true
+      msg-style: capitalized
+      key-naming-case: camel
+formatters:
+  enable:
+    - gofumpt
+  exclusions:
+    generated: lax
+    paths:
+      - internal/gen
+      - cmd/dev
+      - repos
+      - third_party$
+      - builtin$
+      - examples$

.policy.yml

@@ -0,0 +1,111 @@
+# This is the policy-bot configuration for this repository. It controls
+# which approvals are required for any given pull request. The format is
+# described at https://github.com/palantir/policy-bot. The syntax of the
+# policy can be verified by the bot:
+# curl https://pb.syncthing.net/api/validate -X PUT -T .policy.yml
+
+# The policy below is what is required for any pull request.
+policy:
+  approval:
+    - subject is conventional commit
+    - or:
+      - project metadata requires maintainer approval
+      - a maintainer claims responsibility
+    - or:
+      - is approved by a syncthing contributor
+      - is a translation or dependency update by a contributor
+      - is a trivial change by a contributor
+      - a maintainer claims responsibility
+
+  # Additionally, maintainers can disapprove of a PR
+  disapproval:
+    requires:
+      teams:
+        - syncthing/maintainers
+
+# The rules for the policy are described below.
+
+approval_rules:
+
+  # All commits (PRs before squashing) should have a valid conventional
+  # commit type subject.
+  - name: subject is conventional commit
+    requires:
+      conditions:
+        title:
+          matches:
+            - '^(feat|fix|docs|chore|refactor|build): [a-z].+'
+            - '^(feat|fix|docs|chore|refactor|build)\(\w+(, \w+)*\): [a-z].+'
+
+  # Changes to important project metadata and documentation, including this
+  # policy, require signoff by a maintainer
+  - name: project metadata requires maintainer approval
+    if:
+      changed_files:
+        paths:
+          - ^[^/]+\.md
+          - ^\.policy\.yml
+          - ^LICENSE
+    requires:
+      count: 1
+      teams:
+        - syncthing/maintainers
+    options:
+      ignore_update_merges: true
+      allow_non_author_contributor: true
+
+  # Regular pull requests require approval by an active contributor
+  - name: is approved by a syncthing contributor
+    requires:
+      count: 1
+      teams:
+        - syncthing/contributors
+    options:
+      ignore_update_merges: true
+      allow_non_author_contributor: true
+
+  # Changes to some files (translations, dependencies, compatibility) do not
+  # require approval if they were proposed by a contributor and have a
+  # matching commit subject
+  - name: is a translation or dependency update by a contributor
+    if:
+      only_changed_files:
+        paths:
+          - ^gui/default/assets/lang/
+          - ^go\.mod$
+          - ^go\.sum$
+          - ^compat\.yaml$
+      title:
+        matches:
+          - '^chore\(gui\):'
+          - '^build\(deps\):'
+          - '^build\(compat\):'
+      has_author_in:
+        teams:
+          - syncthing/contributors
+
+  # If the change is small and the label "trivial" is added, we accept that
+  # on trust. These PRs can be audited after the fact as appropriate.
+  # Features are not trivial.
+  - name: is a trivial change by a contributor
+    if:
+      modified_lines:
+        total: "< 25"
+      title:
+        not_matches:
+          - '^feat'
+      has_labels:
+        - trivial
+      has_author_in:
+        teams:
+          - syncthing/contributors
+
+  # A member of the maintainers group can take responsibility by adding the
+  # appropriate label.
+  - name: a maintainer claims responsibility
+    if:
+      has_labels:
+        - maintainer-responsibility
+      has_author_in:
+        teams:
+          - syncthing/maintainers

AUTHORS

@@ -13,117 +13,115 @@
 # contents of this file.
 #
 
-Aaron Bieber (qbit) <qbit@deftly.net>
+Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net> <jborg@coreweave.com>
+Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
+Simon Frei (imsodin) <freisim93@gmail.com>
+Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com> <twilczynski@naver.com>
+Alexander Graf (alex2108) <register-github@alex-graf.de>
+Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
+Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
+André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
+Antony Male (canton7) <antony.male@gmail.com>
+Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
+bt90 <btom1990@googlemail.com>
+Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
+Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
+Emil Lundberg <emil@emlun.se>
+Eric P <eric@kastelo.net>
+Evgeny Kuznetsov <evgeny@kuznetsov.md>
+greatroar <61184462+greatroar@users.noreply.github.com>
+Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
+Lode Hoste (Zillode) <zillode@zillode.be>
+Marcus B Spencer <marcus@marcusspencer.xyz> <marcus@marcusspencer.us>
+Michael Ploujnikov (plouj) <ploujj@gmail.com>
+Ross Smith II (rasa) <ross@smithii.com>
+Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
+Tommy van der Vorst <tommy-github@pixelspark.nl> <tommy@pixelspark.nl>
+Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
 Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com> <adam@proactiveservices.co.uk>
 Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
-Alan Pope <alan@popey.com>
-Alberto Donato <albertodonato@users.noreply.github.com>
 Aleksey Vasenev <margtu-fivt@ya.ru>
 Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
+Alex Ionescu <github@ionescu.sh>
 Alex Lindeman <139387+aelindeman@users.noreply.github.com>
 Alex Xu <alex.hello71@gmail.com>
-Alexander Graf (alex2108) <register-github@alex-graf.de>
 Alexander Seiler <seileralex@gmail.com>
 Alexandre Alves <alexandrealvesdb.contact@gmail.com>
-Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
 Aman Gupta <aman@tmm1.net>
-Anatoli Babenia <anatoli@rainforce.org>
-Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
 Andreas Sommer <andreas.sommer87@googlemail.com>
 andresvia <andres.via@gmail.com>
-Andrew Dunham (andrew-d) <andrew@du.nham.ca>
-Andrew Meyer <andrewm.bpi@gmail.com>
 Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
 Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
-André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
 andyleap <andyleap@gmail.com>
 Anjan Momi <anjan@momi.ca>
 Anthony Goeckner <agoeckner@users.noreply.github.com>
 Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
-Antony Male (canton7) <antony.male@gmail.com>
 Anur <anurnomeru@163.com>
 Aranjedeath <Aranjedeath@users.noreply.github.com>
+ardevd <ardevd@users.noreply.github.com>
 Arkadiusz Tymiński <gevleeog@gmail.com>
 Aroun <login@b-vo.fr>
 Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
 Artur Zubilewicz <AkaZecik@users.noreply.github.com>
-Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
+Ashish Bhate <bhate.ashish@gmail.com>
 Aurélien Rainone <476650+arl@users.noreply.github.com>
 BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
 Bart De Vries (mogwa1) <devriesb@gmail.com>
 Beat Reichenbach <44111292+beatreichenbach@users.noreply.github.com>
-Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
-Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
 Ben Shepherd (benshep) <bjashepherd@gmail.com>
 Ben Sidhom (bsidhom) <bsidhom@gmail.com>
 Benedikt Heine (bebehei) <bebe@bebehei.de>
-Benedikt Morbach <benedikt.morbach@googlemail.com>
-Benjamin Nater <17193640+bn4t@users.noreply.github.com>
 Benno Fünfstück <benno.fuenfstueck@gmail.com>
 Benny Ng (tpng) <benny.tpng@gmail.com>
 boomsquared <54829195+boomsquared@users.noreply.github.com>
 Boqin Qin <bobbqqin@bupt.edu.cn>
 Boris Rybalkin <ribalkin@gmail.com>
-Brandon Philips (philips) <brandon@ifup.org>
 Brendan Long (brendanlong) <self@brendanlong.com>
-Brian R. Becker (brbecker) <brbecker@gmail.com>
-bt90 <btom1990@googlemail.com>
-Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
-Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
 Catfriend1 <16361913+Catfriend1@users.noreply.github.com>
 Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
 Cedric Staniewski (xduugu) <cedric@gmx.ca>
-chenrui <rui@meetup.com>
 Chih-Hsuan Yen <yan12125@gmail.com> <1937689+yan12125@users.noreply.github.com>
 Choongkyu <choongkyu.kim+gh@gmail.com> <vapidlyrapid+gh@gmail.com>
 Chris Howie (cdhowie) <me@chrishowie.com>
 Chris Joel (cdata) <chris@scriptolo.gy>
-Chris Tonkinson <chris@masterbran.ch>
 Christian Kujau <ckujau@users.noreply.github.com>
 Christian Prescott <me@christianprescott.com>
 chucic <chucic@seznam.cz>
 cjc7373 <niuchangcun@gmail.com>
 Colin Kennedy (moshen) <moshen.colin@gmail.com>
 Cromefire_ <tim.l@nghorst.net> <26320625+cromefire@users.noreply.github.com>
-cui fliter <imcusg@gmail.com>
 Cyprien Devillez <cypx@users.noreply.github.com>
 d-volution <49024624+d-volution@users.noreply.github.com>
-Dale Visser <dale.visser@live.com>
 Dan <benda.daniel@gmail.com>
 Daniel Barczyk <46358936+DanielBarczyk@users.noreply.github.com>
 Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
-Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
 Daniel Martí (mvdan) <mvdan@mvdan.cc>
 Daniel Padrta <64928366+danpadcz@users.noreply.github.com>
+Daniil Gentili <daniil@daniil.it>
 Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
+dashangcun <907225865@qq.com>
 David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
-deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com>
 DeflateAwning <11021263+DeflateAwning@users.noreply.github.com>
 Denis A. (dva) <denisva@gmail.com>
 Dennis Wilson (snnd) <dw@risu.io>
-dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com> <27856297+dependabot-preview[bot]@users.noreply.github.com>
-dependabot[bot] <dependabot[bot]@users.noreply.github.com> <49699333+dependabot[bot]@users.noreply.github.com>
 derekriemer <derek.riemer@colorado.edu>
 DerRockWolf <50499906+DerRockWolf@users.noreply.github.com>
 desbma <desbma@users.noreply.github.com>
 Devon G. Redekopp <devon@redekopp.com>
-diemade <spamkill@posteo.ch>
 digital <didev@dinid.net>
 Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com>
 Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
+domain <32405309+szu17dmy@users.noreply.github.com>
 Domenic Horner <domenic@tgxn.net>
 Dominik Heidler (asdil12) <dominik@heidler.eu>
 Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
 Elliot Huffman <thelich2@gmail.com>
 Emil Hessman (ceh) <emil@hessman.se>
-Emil Lundberg <emil@emlun.se>
 Eng Zer Jun <engzerjun@gmail.com>
 entity0xfe <109791748+entity0xfe@users.noreply.github.com> <entity0xfe@my.domain>
 Eric Lesiuta <elesiuta@gmail.com>
-Eric P <eric@kastelo.net>
 Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
 Evan Spensley <94762716+0evan@users.noreply.github.com>
-Evgeny Kuznetsov <evgeny@kuznetsov.md>
 Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
 Felix <53702818+f-eliks@users.noreply.github.com>
 Felix Ableitner (Nutomic) <me@nutomic.com>
@@ -137,7 +135,6 @@ ghjklw <malo@jaffre.info>
 Gilli Sigurdsson (gillisig) <gilli@vx.is>
 Gleb Sinyavskiy <zhulik.gleb@gmail.com>
 Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
-greatroar <61184462+greatroar@users.noreply.github.com>
 Greg <gco@jazzhaiku.com>
 guangwu <guoguangwu@magic-shield.com>
 gudvinr <gudvinr@gmail.com>
@@ -145,54 +142,42 @@ Gusted <postmaster@gusted.xyz> <williamzijl7@hotmail.com>
 Han Boetes <han@boetes.org>
 HansK-p <42314815+HansK-p@users.noreply.github.com>
 Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
+Hazem Krimi <me@hazemkrimi.tech>
 Heiko Zuerker (Smiley73) <heiko@zuerker.org>
+Hireworks <129852174+hireworksltd@users.noreply.github.com>
 Hugo Locurcio <hugo.locurcio@hugo.pro>
 Iain Barnett <iainspeed@gmail.com>
 Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
 ignacy123 <ignacy.buczek@onet.pl>
-Ikko Ashimine <eltociear@gmail.com>
-Ilya Brin <464157+ilyabrin@users.noreply.github.com>
 Iskander Sharipov (Alex) <quasilyte@gmail.com>
 Jaakko Hannikainen (jgke) <jgke@jgke.fi>
-Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
 Jack Croft <jccroft1@users.noreply.github.com>
 Jacob <jyundt@gmail.com>
 Jake Peterson (acogdev) <jake@acogdev.com>
-Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net> <jborg@coreweave.com>
 James O'Beirne <wild-github@au92.org>
 James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
-janost <janost@tuta.io>
 Jaroslav Lichtblau <svetlemodry@users.noreply.github.com>
 Jaroslav Malec (dzarda) <dzardacz@gmail.com>
-jaseg <githubaccount@jaseg.net>
 Jaspitta <ste.scarpitta@gmail.com>
-Jauder Ho <jauderho@users.noreply.github.com>
 Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
 Jaya Kumar <jaya.kumar@ict.nl>
 Jeffery To <jeffery.to@gmail.com>
 jelle van der Waa <jelle@vdwaa.nl>
 Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
-Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
-Jesse Lucas <jesse@jesselucas.com>
 Jochen Voss (seehuhn) <voss@seehuhn.de>
-Johan Andersson <j@i19.se>
 Johan Vromans (sciurius) <jvromans@squirrel.nl>
 John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
 Jonas Thelemann <e-mail@jonas-thelemann.de>
 Jonathan <artback@protonmail.com> <jonagn@gmail.com>
-Jonathan Cross <jcross@gmail.com>
-Jonta <359397+Jonta@users.noreply.github.com>
 Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
 jtagcat <git-514635f7@jtag.cat> <git-12dbd862@jtag.cat>
 Julian Lehrhuber <jul13579@users.noreply.github.com>
 Jörg Thalheim <Mic92@users.noreply.github.com>
 Jędrzej Kula <kula.jedrek@gmail.com>
-K.B.Dharun Krishna <kbdharunkrishna@gmail.com>
-Kalle Laine <pahakalle@protonmail.com>
+Kapil Sareen <kapilsareen584@gmail.com>
 Karol Różycki (krozycki) <rozycki.karol@gmail.com>
 Kebin Liu <lkebin@gmail.com>
 Keith Harrison <keithh@protonmail.com>
-Keith Turner <kturner@apache.org>
 Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
 Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
 Kevin Allen (ironmig) <kma1660@gmail.com>
@@ -201,31 +186,26 @@ Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
 klemens <ka7@github.com>
 Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
 kylosus <33132401+kylosus@users.noreply.github.com>
-Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
 Lars Lehtonen <lars.lehtonen@gmail.com>
-Laurent Arnoud <laurent@spkdev.net>
 Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
 Leo Arias (elopio) <yo@elopio.net>
 Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
-Lode Hoste (Zillode) <zillode@zillode.be>
 Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
 LSmithx2 <42276854+lsmithx2@users.noreply.github.com>
-luchenhan <168071714+luchenhan@users.noreply.github.com>
 Lukas Lihotzki <lukas@lihotzki.de>
 Luke Hamburg <1992842+luckman212@users.noreply.github.com>
 luzpaz <luzpaz@users.noreply.github.com>
 Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
 Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
-Marc Pujol (kilburn) <kilburn@la3.org>
+Marcel Meyer <mm.marcelmeyer@gmail.com>
 Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
-marco-m <marco.molteni@laposte.net>
 Marcus Legendre <marcus.legendre@gmail.com>
 Mario Majila <mariustshipichik@gmail.com>
 Mark Pulford (mpx) <mark@kyne.com.au>
 Martchus <martchus@gmx.net>
-Martin Polehla <p0l0us@users.noreply.github.com>
 Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
 Mateusz Ż <thedead4fun@live.com>
+mathias4833 <67101597+mathias4833@users.noreply.github.com>
 Matic Potočnik <hairyfotr@gmail.com>
 Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
 Matt Robenolt <matt@ydekproductions.com>
@@ -234,13 +214,9 @@ Maurizio Tomasi <ziotom78@gmail.com>
 Max <github@germancoding.com>
 Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
 MaximAL <almaximal@ya.ru>
-Maxime Thirouin <m@moox.io>
 Maximilian <maxi.rostock@outlook.de> <public@complexvector.space>
-mclang <1721600+mclang@users.noreply.github.com>
 Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
-Michael Ploujnikov (plouj) <ploujj@gmail.com>
 Michael Rienstra <mrienstra@gmail.com>
-Michael Tilli (pyfisch) <pyfisch@gmail.com>
 MichaIng <micha@dietpi.com>
 Migelo <miha@filetki.si>
 Mike Boone <mike@boonedocks.net>
@@ -249,7 +225,6 @@ MikolajTwarog <43782609+MikolajTwarog@users.noreply.github.com>
 Mingxuan Lin <gdlmx@users.noreply.github.com>
 mv1005 <49659413+mv1005@users.noreply.github.com>
 Nate Morrison (nrm21) <natemorrison@gmail.com>
-Naveen <172697+naveensrinivasan@users.noreply.github.com>
 nf <nf@wh3rd.net>
 Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
 Nick Busey <NickBusey@users.noreply.github.com>
@@ -264,14 +239,13 @@ NoLooseEnds <jon.koslung@gmail.com>
 Oliver Freyermuth <o.freyermuth@googlemail.com>
 orangekame3 <miya.org.0309@gmail.com>
 otbutz <tbutz@optitool.de>
-Otiel <Otiel@users.noreply.github.com>
 overkill <22098433+0verk1ll@users.noreply.github.com>
 Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
 Pablo <pbaeyens31+github@gmail.com>
 Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
 Paul Brit <paulbrit44@gmail.com>
+Paul Donald <newtwen+github@gmail.com>
 Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
-Paweł Rozlach <vespian@users.noreply.github.com>
 perewa <cavalcante.ten@gmail.com>
 Peter Badida <KeyWeeUsr@users.noreply.github.com>
 Peter Dave Hello <hsu@peterdavehello.org>
@@ -281,18 +255,16 @@ Phani Rithvij <phanirithvij2000@gmail.com>
 Phil Davis <phil.davis@inf.org>
 Philippe Schommers (filoozoom) <philippe@schommers.be>
 Phill Luby (pluby) <phill.luby@newredo.com>
-Pier Paolo Ramon <ramonpierre@gmail.com>
 Piotr Bejda (piobpl) <piotrb10@gmail.com>
-Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
+polyfloyd <polyfloyd@users.noreply.github.com>
+pullmerge <166967364+pullmerge@users.noreply.github.com>
 Quentin Hibon <qh.public@yahoo.com>
 Rahmi Pruitt <rjpruitt16@gmail.com>
 red_led <red-led@users.noreply.github.com>
-Richard Hartmann <RichiH@users.noreply.github.com>
 Robert Carosi (nov1n) <robert@carosi.nl>
 Roberto Santalla <roobre@users.noreply.github.com>
 Robin Schoonover <robin@cornhooves.org>
 Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
-Ross Smith II (rasa) <ross@smithii.com>
 rubenbe <github-com-00ff86@vandamme.email>
 Ruslan Yevdokymov <38809160+ruslanye@users.noreply.github.com>
 Ryan Qian <i@bitbili.net>
@@ -304,31 +276,27 @@ Sergey Mishin (ralder) <ralder@yandex.ru>
 Sertonix <83883937+Sertonix@users.noreply.github.com>
 Severin von Wnuck-Lipinski <ss7@live.de>
 Shaarad Dalvi <60266155+shaaraddalvi@users.noreply.github.com> <shdalv@microsoft.com>
-Simon Frei (imsodin) <freisim93@gmail.com>
 Simon Mwepu <simonmwepu@gmail.com>
 Simon Pickup <simon@pickupinfinity.com>
 Sly_tom_cat <slytomcat@mail.ru>
 Sonu Kumar Saw <31889738+dev-saw99@users.noreply.github.com>
 Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
-Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
 Steven Eckhoff <steven.eckhoff.opensource@gmail.com>
 Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
 Sven Bachmann <dev@mcbachmann.de>
-Syncthing Automation <automation@syncthing.net>
-Syncthing Release Automation <release@syncthing.net>
+Sébastien WENSKE <sebastien@wenske.fr>
 Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
+Terrance <git@terrance.allofti.me>
+TheCreeper <TheCreeper@users.noreply.github.com>
 Thomas <9749173+uhthomas@users.noreply.github.com>
 Thomas Hipp <thomashipp@gmail.com>
 Tim Abell (timabell) <tim@timwise.co.uk>
 Tim Howes (timhowes) <timhowes@berkeley.edu>
-Tim Nordenfur <tim@gurka.se>
+Tobias Frölich <40638719+tobifroe@users.noreply.github.com>
 Tobias Klauser <tobias.klauser@gmail.com>
 Tobias Nygren (tnn2) <tnn@nygren.pp.se>
 Tobias Tom (tobiastom) <t.tom@succont.de>
 Tom Jakubowski <tom@crystae.net>
-Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com> <twilczynski@naver.com>
-Tommy Thorn <tommy-github-email@thorn.ws>
-Tommy van der Vorst <tommy-github@pixelspark.nl> <tommy@pixelspark.nl>
 Tully Robinson (tojrobinson) <tully@tojr.org>
 Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
 Tyler Kropp <kropptyler@gmail.com>
@@ -345,10 +313,10 @@ WangXi <xib1102@icloud.com>
 Will Rouesnel <wrouesnel@wrouesnel.com>
 William A. Kennington III (wkennington) <william@wkennington.com>
 wouter bolsterlee <wouter@bolsterl.ee>
-Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
 xarx00 <xarx00@users.noreply.github.com>
 Xavier O. (damajor) <damajor@gmail.com>
-xjtdy888 (xjtdy888) <xjtdy888@163.com>
+xjtdy888 (xjtdy888) <xjtdy888@163.com> <xjtdy888@gmail.com>
 Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
+yparitcher <y@paritcher.com>
 佛跳墙 <daoquan@qq.com>
 落心 <luoxin.ttt@gmail.com>

CONTRIBUTING.md

@@ -34,19 +34,163 @@ Note that the previously used service at
 retired and we kindly ask you to sign up on Weblate for continued
 involvement.
 
-## Contributing Code
-
-Every contribution is welcome. If you want to contribute but are unsure
-where to start, any open issues are fair game! See the [Contribution
-Guidelines](https://docs.syncthing.net/dev/contributing.html) for the full
-story on committing code.
-
 ## Contributing Documentation
 
 Updates to the [documentation site](https://docs.syncthing.net/) can be
 made as pull requests on the [documentation
 repository](https://github.com/syncthing/docs).
 
+## Contributing Code
+
+Every contribution is welcome. If you want to contribute but are unsure
+where to start, any open issues are fair game! Here's a short rundown of
+what you need to keep in mind:
+
+- Don't worry. You are not expected to get everything right on the first
+  attempt, we'll guide you through it.
+
+- Make sure there is an
+  [issue](https://github.com/syncthing/syncthing/issues) that describes the
+  change you want to do. If the thing you want to do does not have an issue
+  yet, please file one before starting work on it.
+
+- Fork the repository and make your changes in a new branch. Once it's ready
+  for review, create a pull request.
+
+### Authorship
+
+All code authors are listed in the AUTHORS file. When your first pull
+request is accepted your details are added to the AUTHORS file and the list
+of authors in the GUI. Commits must be made with the same name and email as
+listed in the AUTHORS file. To accomplish this, ensure that your git
+configuration is set correctly prior to making your first commit:
+
+    $ git config --global user.name "Jane Doe"
+    $ git config --global user.email janedoe@example.com
+
+You must be reachable on the given email address. If you do not wish to use
+your real name for whatever reason, using a nickname or pseudonym is
+perfectly acceptable.
+
+### The Developer Certificate of Origin (DCO)
+
+The Syncthing project requires the Developer Certificate of Origin (DCO)
+sign-off on pull requests (PRs). This means that all commit messages must
+contain a signature line to indicate that the developer accepts the DCO.
+
+The DCO is a lightweight way for contributors to certify that they wrote (or
+otherwise have the right to submit) the code and changes they are
+contributing to the project. Here is the full [text of the
+DCO](https://developercertificate.org):
+
+---
+
+By making a contribution to this project, I certify that:
+
+1. The contribution was created in whole or in part by me and I have the
+   right to submit it under the open source license indicated in the file;
+   or
+
+2. The contribution is based upon previous work that, to the best of my
+   knowledge, is covered under an appropriate open source license and I have
+   the right under that license to submit that work with modifications,
+   whether created in whole or in part by me, under the same open source
+   license (unless I am permitted to submit under a different license), as
+   indicated in the file; or
+
+3. The contribution was provided directly to me by some other person who
+   certified (1), (2) or (3) and I have not modified it.
+
+4. I understand and agree that this project and the contribution are public
+   and that a record of the contribution (including all personal information
+   I submit with it, including my sign-off) is maintained indefinitely and
+   may be redistributed consistent with this project or the open source
+   license(s) involved.
+
+---
+
+Contributors indicate that they adhere to these requirements by adding
+a `Signed-off-by` line to their commit messages.  For example:
+
+    This is my commit message
+
+    Signed-off-by: Random J Developer <random@developer.example.org>
+
+The name and email address in this line must match those of the committing
+author, and be the same as what you want in the AUTHORS file as per above.
+
+### Coding Style
+
+#### General
+
+- All text files use Unix line endings. The git settings already present in
+  the repository attempt to enforce this.
+
+- When making changes, follow the brace and parenthesis style of the
+  surrounding code.
+
+#### Go Specific
+
+- Follow the conventions laid out in [Effective
+  Go](https://go.dev/doc/effective_go) as much as makes sense. The review
+  guidelines in [Go Code Review
+  Comments](https://github.com/golang/go/wiki/CodeReviewComments) should
+  generally be followed.
+
+- Each commit should be `go fmt` clean.
+
+- Imports are grouped per `goimports` standard; that is, standard
+  library first, then third party libraries after a blank line.
+
+### Commits
+
+- Commit messages (and pull request titles) should follow the [conventional
+  commits](https://www.conventionalcommits.org/en/v1.0.0/) specification and
+  be in lower case.
+
+- We use a scope description in the commit message subject. This is the
+  component of Syncthing that the commit affects. For example, `gui`,
+  `protocol`, `scanner`, `upnp`, etc -- typically, the part after
+  `internal/`, `lib/` or `cmd/` in the package path. If the commit doesn't
+  affect a specific component, such as for changes to the build system or
+  documentation, the scope should be omitted. The same goes for changes that
+  affect many components which would be cumbersome to list.
+
+- Commits that resolve an existing issue must include the issue number
+  as `(fixes #123)` at the end of the commit message subject. A correctly
+  formatted commit message subject looks like this:
+
+      feat(dialer): add env var to disable proxy fallback (fixes #3006)
+
+- If the commit message subject doesn't say it all, one or more paragraphs of
+  describing text should be added to the commit message. This should explain
+  why the change is made and what it accomplishes.
+
+- When drafting a pull request, please feel free to add commits with
+  corrections and merge from `main` when necessary. This provides a clear time
+  line with changes and simplifies review. Do not, in general, rebase your
+  commits, as this makes review harder.
+
+- Pull requests are merged to `main` using squash merge. The "stream of
+  consciousness" set of commits described in the previous point will be reduced
+  to a single commit at merge time. The pull request title and description will
+  be used as the commit message.
+
+### Tests
+
+Yes please, do add tests when adding features or fixing bugs. Also, when a
+pull request is filed a number of automatic tests are run on the code. This
+includes:
+
+- That the code actually builds and the test suite passes.
+
+- That the code is correctly formatted (`go fmt`).
+
+- That the commits are based on a reasonably recent `main`.
+
+- That the output from `go lint` and `go vet` is clean. (This checks for a
+  number of potential problems the compiler doesn't catch.)
+
 ## Licensing
 
 All contributions are made available under the same license as the already
@@ -59,10 +203,6 @@ otherwise stated this means MPLv2, but there are exceptions:
 - The documentation (man/...) is licensed under the Creative Commons
   Attribution 4.0 International License.
 
-- Projects under vendor/... are copyright by and licensed from their
-  respective original authors. Contributions should be made to the original
-  project, not here.
-
 Regardless of the license in effect, you retain the copyright to your
 contribution.
 

README-Docker.md

@@ -23,47 +23,7 @@ example `UMASK=002`.
 **Docker cli**
 ```
 $ docker pull syncthing/syncthing
-$ docker run -p 8384:8384 -p 22000:22000/tcp -p 22000:22000/udp -p 21027:21027/udp \
-    -v /wherever/st-sync:/var/syncthing \
-    --hostname=my-syncthing \
-    syncthing/syncthing:latest
-```
-
-**Docker compose**
-```yml
----
-version: "3"
-services:
-  syncthing:
-    image: syncthing/syncthing
-    container_name: syncthing
-    hostname: my-syncthing
-    environment:
-      - PUID=1000
-      - PGID=1000
-    volumes:
-      - /wherever/st-sync:/var/syncthing
-    ports:
-      - 8384:8384 # Web UI
-      - 22000:22000/tcp # TCP file transfers
-      - 22000:22000/udp # QUIC file transfers
-      - 21027:21027/udp # Receive local discovery broadcasts
-    restart: unless-stopped
-```
-
-## Discovery
-
-Note that Docker's default network mode prevents local IP addresses from
-being discovered, as Syncthing is only able to see the internal IP of the
-container on the `172.17.0.0/16` subnet. This will result in poor transfer rates
-if local device addresses are not manually configured.
-
-It is therefore advisable to use the [host network mode](https://docs.docker.com/network/host/) instead:
-
-**Docker cli**
-```
-$ docker pull syncthing/syncthing
-$ docker run --network=host \
+$ docker run --network=host  -e STGUIADDRESS= \
     -v /wherever/st-sync:/var/syncthing \
     syncthing/syncthing:latest
 ```
@@ -80,33 +40,39 @@ services:
     environment:
       - PUID=1000
       - PGID=1000
+      - STGUIADDRESS=
     volumes:
       - /wherever/st-sync:/var/syncthing
     network_mode: host
     restart: unless-stopped
+    healthcheck:
+      test: curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -o --color=never OK || exit 1
+      interval: 1m
+      timeout: 10s
+      retries: 3
 ```
 
+## Discovery
+
+Please note that Docker's default network mode prevents local IP addresses
+from being discovered, as Syncthing can only see the internal IP address of
+the container on the `172.17.0.0/16` subnet. This would likely break the ability
+for nodes to establish LAN connections properly, resulting in poor transfer
+rates unless local device addresses are configured manually.
+
+It is therefore strongly recommended to stick to the [host network mode](https://docs.docker.com/network/host/),
+as shown above.
+
 Be aware that syncthing alone is now in control of what interfaces and ports it
 listens on. You can edit the syncthing configuration to change the defaults if
 there are conflicts.
 
 ## GUI Security
 
-By default Syncthing inside the Docker image listens on 0.0.0.0:8384 to
-allow GUI connections via the Docker proxy. This is set by the
-`STGUIADDRESS` environment variable in the Dockerfile, as it differs from
-what Syncthing would otherwise use by default. This means you should set up
-authentication in the GUI, like for any other externally reachable Syncthing
-instance. If you do not require the GUI, or you use host networking, you can
-unset the `STGUIADDRESS` variable to have Syncthing fall back to listening
-on 127.0.0.1:
-
-```
-$ docker pull syncthing/syncthing
-$ docker run -e STGUIADDRESS= \
-    -v /wherever/st-sync:/var/syncthing \
-    syncthing/syncthing:latest
-```
-
-With the environment variable unset Syncthing will follow what is set in the
-configuration file / GUI settings dialog.
+By default Syncthing inside the Docker image listens on `0.0.0.0:8384`. This
+allows GUI connections when running without host network mode. The example
+above unsets the `STGUIADDRESS` environment variable to have Syncthing fall
+back to listening on what has been configured in the configuration file or the
+GUI settings dialog. By default this is the localhost IP address `127.0.0.1`.
+If you configure your GUI to be externally reachable, make sure you set up
+authentication and enable TLS.

README.md

@@ -82,13 +82,11 @@ build process.
 
 ## Signed Releases
 
-As of v0.10.15 and onwards, release binaries are GPG signed with the key
-D26E6ED000654A3E, available from https://syncthing.net/security/ and
-most key servers.
-
-There is also a built-in automatic upgrade mechanism (disabled in some
-distribution channels) which uses a compiled in ECDSA signature. macOS
-binaries are also properly code signed.
+Release binaries are GPG signed with the key available from
+https://syncthing.net/security/. There is also a built-in automatic
+upgrade mechanism (disabled in some distribution channels) which uses a
+compiled in ECDSA signature. macOS and Windows binaries are also
+code-signed.
 
 ## Documentation
 

buf.gen.yaml

@@ -0,0 +1,12 @@
+version: v2
+managed:
+  enabled: true
+  override:
+    - file_option: go_package_prefix
+      value: github.com/syncthing/syncthing/internal/gen
+plugins:
+  - remote: buf.build/protocolbuffers/go:v1.35.1
+    out: .
+    opt: module=github.com/syncthing/syncthing
+inputs:
+  - directory: proto

buf.yaml

@@ -0,0 +1,10 @@
+version: v2
+modules:
+  - path: proto
+    name: github.com/syncthing/syncthing
+lint:
+  use:
+    - STANDARD
+breaking:
+  use:
+    - WIRE_JSON

build.go

@@ -15,7 +15,6 @@ import (
 	"bytes"
 	"compress/flate"
 	"compress/gzip"
-	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"flag"
@@ -39,27 +38,26 @@ import (
 )
 
 var (
-	goarch         string
-	goos           string
-	noupgrade      bool
-	version        string
-	goCmd          string
-	race           bool
-	debug          = os.Getenv("BUILDDEBUG") != ""
-	extraTags      string
-	installSuffix  string
-	pkgdir         string
-	cc             string
-	run            string
-	benchRun       string
-	buildOut       string
-	debugBinary    bool
-	coverage       bool
-	long           bool
-	timeout        = "120s"
-	longTimeout    = "600s"
-	numVersions    = 5
-	withNextGenGUI = os.Getenv("BUILD_NEXT_GEN_GUI") != ""
+	goarch        string
+	goos          string
+	noupgrade     bool
+	version       string
+	goCmd         string
+	race          bool
+	debug         = os.Getenv("BUILDDEBUG") != ""
+	extraTags     string
+	installSuffix string
+	pkgdir        string
+	cc            string
+	run           string
+	benchRun      string
+	buildOut      string
+	debugBinary   bool
+	coverage      bool
+	long          bool
+	timeout       = "120s"
+	longTimeout   = "600s"
+	numVersions   = 5
 )
 
 type target struct {
@@ -86,7 +84,6 @@ var targets = map[string]target{
 	"all": {
 		// Only valid for the "build" and "install" commands as it lacks all
 		// the archive creation stuff. buildPkgs gets filled out in init()
-		tags: []string{"purego"},
 	},
 	"syncthing": {
 		// The default target for "build", "install", "tar", "zip", "deb", etc.
@@ -97,40 +94,40 @@ var targets = map[string]target{
 		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/syncthing"},
 		binaryName:  "syncthing", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
-			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
-			{src: "README.md", dst: "README.txt", perm: 0644},
-			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
-			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
+			{src: "{{binary}}", dst: "{{binary}}", perm: 0o755},
+			{src: "README.md", dst: "README.txt", perm: 0o644},
+			{src: "LICENSE", dst: "LICENSE.txt", perm: 0o644},
+			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0o644},
 			// All files from etc/ and extra/ added automatically in init().
 		},
 		systemdService: "syncthing@*.service",
 		installationFiles: []archiveFile{
-			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
-			{src: "README.md", dst: "deb/usr/share/doc/syncthing/README.txt", perm: 0644},
-			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing/LICENSE.txt", perm: 0644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing/AUTHORS.txt", perm: 0644},
-			{src: "man/syncthing.1", dst: "deb/usr/share/man/man1/syncthing.1", perm: 0644},
-			{src: "man/syncthing-config.5", dst: "deb/usr/share/man/man5/syncthing-config.5", perm: 0644},
-			{src: "man/syncthing-stignore.5", dst: "deb/usr/share/man/man5/syncthing-stignore.5", perm: 0644},
-			{src: "man/syncthing-device-ids.7", dst: "deb/usr/share/man/man7/syncthing-device-ids.7", perm: 0644},
-			{src: "man/syncthing-event-api.7", dst: "deb/usr/share/man/man7/syncthing-event-api.7", perm: 0644},
-			{src: "man/syncthing-faq.7", dst: "deb/usr/share/man/man7/syncthing-faq.7", perm: 0644},
-			{src: "man/syncthing-networking.7", dst: "deb/usr/share/man/man7/syncthing-networking.7", perm: 0644},
-			{src: "man/syncthing-rest-api.7", dst: "deb/usr/share/man/man7/syncthing-rest-api.7", perm: 0644},
-			{src: "man/syncthing-security.7", dst: "deb/usr/share/man/man7/syncthing-security.7", perm: 0644},
-			{src: "man/syncthing-versioning.7", dst: "deb/usr/share/man/man7/syncthing-versioning.7", perm: 0644},
-			{src: "etc/linux-systemd/system/syncthing@.service", dst: "deb/lib/systemd/system/syncthing@.service", perm: 0644},
-			{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0644},
-			{src: "etc/linux-sysctl/30-syncthing.conf", dst: "deb/usr/lib/sysctl.d/30-syncthing.conf", perm: 0644},
-			{src: "etc/firewall-ufw/syncthing", dst: "deb/etc/ufw/applications.d/syncthing", perm: 0644},
-			{src: "etc/linux-desktop/syncthing-start.desktop", dst: "deb/usr/share/applications/syncthing-start.desktop", perm: 0644},
-			{src: "etc/linux-desktop/syncthing-ui.desktop", dst: "deb/usr/share/applications/syncthing-ui.desktop", perm: 0644},
-			{src: "assets/logo-32.png", dst: "deb/usr/share/icons/hicolor/32x32/apps/syncthing.png", perm: 0644},
-			{src: "assets/logo-64.png", dst: "deb/usr/share/icons/hicolor/64x64/apps/syncthing.png", perm: 0644},
-			{src: "assets/logo-128.png", dst: "deb/usr/share/icons/hicolor/128x128/apps/syncthing.png", perm: 0644},
-			{src: "assets/logo-256.png", dst: "deb/usr/share/icons/hicolor/256x256/apps/syncthing.png", perm: 0644},
-			{src: "assets/logo-512.png", dst: "deb/usr/share/icons/hicolor/512x512/apps/syncthing.png", perm: 0644},
-			{src: "assets/logo-only.svg", dst: "deb/usr/share/icons/hicolor/scalable/apps/syncthing.svg", perm: 0644},
+			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0o755},
+			{src: "README.md", dst: "deb/usr/share/doc/syncthing/README.txt", perm: 0o644},
+			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing/LICENSE.txt", perm: 0o644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing/AUTHORS.txt", perm: 0o644},
+			{src: "man/syncthing.1", dst: "deb/usr/share/man/man1/syncthing.1", perm: 0o644},
+			{src: "man/syncthing-config.5", dst: "deb/usr/share/man/man5/syncthing-config.5", perm: 0o644},
+			{src: "man/syncthing-stignore.5", dst: "deb/usr/share/man/man5/syncthing-stignore.5", perm: 0o644},
+			{src: "man/syncthing-device-ids.7", dst: "deb/usr/share/man/man7/syncthing-device-ids.7", perm: 0o644},
+			{src: "man/syncthing-event-api.7", dst: "deb/usr/share/man/man7/syncthing-event-api.7", perm: 0o644},
+			{src: "man/syncthing-faq.7", dst: "deb/usr/share/man/man7/syncthing-faq.7", perm: 0o644},
+			{src: "man/syncthing-networking.7", dst: "deb/usr/share/man/man7/syncthing-networking.7", perm: 0o644},
+			{src: "man/syncthing-rest-api.7", dst: "deb/usr/share/man/man7/syncthing-rest-api.7", perm: 0o644},
+			{src: "man/syncthing-security.7", dst: "deb/usr/share/man/man7/syncthing-security.7", perm: 0o644},
+			{src: "man/syncthing-versioning.7", dst: "deb/usr/share/man/man7/syncthing-versioning.7", perm: 0o644},
+			{src: "etc/linux-systemd/system/syncthing@.service", dst: "deb/lib/systemd/system/syncthing@.service", perm: 0o644},
+			{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0o644},
+			{src: "etc/linux-sysctl/30-syncthing.conf", dst: "deb/usr/lib/sysctl.d/30-syncthing.conf", perm: 0o644},
+			{src: "etc/firewall-ufw/syncthing", dst: "deb/etc/ufw/applications.d/syncthing", perm: 0o644},
+			{src: "etc/linux-desktop/syncthing-start.desktop", dst: "deb/usr/share/applications/syncthing-start.desktop", perm: 0o644},
+			{src: "etc/linux-desktop/syncthing-ui.desktop", dst: "deb/usr/share/applications/syncthing-ui.desktop", perm: 0o644},
+			{src: "assets/logo-32.png", dst: "deb/usr/share/icons/hicolor/32x32/apps/syncthing.png", perm: 0o644},
+			{src: "assets/logo-64.png", dst: "deb/usr/share/icons/hicolor/64x64/apps/syncthing.png", perm: 0o644},
+			{src: "assets/logo-128.png", dst: "deb/usr/share/icons/hicolor/128x128/apps/syncthing.png", perm: 0o644},
+			{src: "assets/logo-256.png", dst: "deb/usr/share/icons/hicolor/256x256/apps/syncthing.png", perm: 0o644},
+			{src: "assets/logo-512.png", dst: "deb/usr/share/icons/hicolor/512x512/apps/syncthing.png", perm: 0o644},
+			{src: "assets/logo-only.svg", dst: "deb/usr/share/icons/hicolor/scalable/apps/syncthing.svg", perm: 0o644},
 		},
 	},
 	"stdiscosrv": {
@@ -142,23 +139,22 @@ var targets = map[string]target{
 		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/stdiscosrv"},
 		binaryName:  "stdiscosrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
-			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
-			{src: "cmd/stdiscosrv/README.md", dst: "README.txt", perm: 0644},
-			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
-			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
+			{src: "{{binary}}", dst: "{{binary}}", perm: 0o755},
+			{src: "cmd/stdiscosrv/README.md", dst: "README.txt", perm: 0o644},
+			{src: "LICENSE", dst: "LICENSE.txt", perm: 0o644},
+			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0o644},
 		},
 		systemdService: "stdiscosrv.service",
 		installationFiles: []archiveFile{
-			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
-			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0644},
-			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0644},
-			{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0644},
-			{src: "cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service", dst: "deb/lib/systemd/system/stdiscosrv.service", perm: 0644},
-			{src: "cmd/stdiscosrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-discosrv", perm: 0644},
-			{src: "cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv", dst: "deb/etc/ufw/applications.d/stdiscosrv", perm: 0644},
+			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0o755},
+			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0o644},
+			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0o644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0o644},
+			{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0o644},
+			{src: "cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service", dst: "deb/lib/systemd/system/stdiscosrv.service", perm: 0o644},
+			{src: "cmd/stdiscosrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-discosrv", perm: 0o644},
+			{src: "cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv", dst: "deb/etc/ufw/applications.d/stdiscosrv", perm: 0o644},
 		},
-		tags: []string{"purego"},
 	},
 	"strelaysrv": {
 		name:        "strelaysrv",
@@ -169,44 +165,30 @@ var targets = map[string]target{
 		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaysrv"},
 		binaryName:  "strelaysrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
-			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
-			{src: "cmd/strelaysrv/README.md", dst: "README.txt", perm: 0644},
-			{src: "cmd/strelaysrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
-			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
-			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
+			{src: "{{binary}}", dst: "{{binary}}", perm: 0o755},
+			{src: "cmd/strelaysrv/README.md", dst: "README.txt", perm: 0o644},
+			{src: "cmd/strelaysrv/LICENSE", dst: "LICENSE.txt", perm: 0o644},
+			{src: "LICENSE", dst: "LICENSE.txt", perm: 0o644},
+			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0o644},
 		},
 		systemdService: "strelaysrv.service",
 		installationFiles: []archiveFile{
-			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
-			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0644},
-			{src: "cmd/strelaysrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
-			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0644},
-			{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0644},
-			{src: "cmd/strelaysrv/etc/linux-systemd/strelaysrv.service", dst: "deb/lib/systemd/system/strelaysrv.service", perm: 0644},
-			{src: "cmd/strelaysrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-relaysrv", perm: 0644},
-			{src: "cmd/strelaysrv/etc/firewall-ufw/strelaysrv", dst: "deb/etc/ufw/applications.d/strelaysrv", perm: 0644},
+			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0o755},
+			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0o644},
+			{src: "cmd/strelaysrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0o644},
+			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0o644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0o644},
+			{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0o644},
+			{src: "cmd/strelaysrv/etc/linux-systemd/strelaysrv.service", dst: "deb/lib/systemd/system/strelaysrv.service", perm: 0o644},
+			{src: "cmd/strelaysrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-relaysrv", perm: 0o644},
+			{src: "cmd/strelaysrv/etc/firewall-ufw/strelaysrv", dst: "deb/etc/ufw/applications.d/strelaysrv", perm: 0o644},
 		},
 	},
 	"strelaypoolsrv": {
 		name:        "strelaypoolsrv",
-		debname:     "syncthing-relaypoolsrv",
-		debdeps:     []string{"libc6"},
 		description: "Syncthing Relay Pool Server",
 		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/infra/strelaypoolsrv"},
-		binaryName:  "strelaypoolsrv", // .exe will be added automatically for Windows builds
-		archiveFiles: []archiveFile{
-			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
-			{src: "cmd/infra/strelaypoolsrv/README.md", dst: "README.txt", perm: 0644},
-			{src: "cmd/infra/strelaypoolsrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
-			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
-		},
-		installationFiles: []archiveFile{
-			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
-			{src: "cmd/infra/strelaypoolsrv/README.md", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/README.txt", perm: 0644},
-			{src: "cmd/infra/strelaypoolsrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/LICENSE.txt", perm: 0644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/AUTHORS.txt", perm: 0644},
-		},
+		binaryName:  "strelaypoolsrv",
 	},
 	"stupgrades": {
 		name:        "stupgrades",
@@ -244,13 +226,13 @@ func initTargets() {
 	// and "extra" dirs.
 	syncthingPkg := targets["syncthing"]
 	for _, file := range listFiles("etc") {
-		syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0644})
+		syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0o644})
 	}
 	for _, file := range listFiles("extra") {
-		syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0644})
+		syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0o644})
 	}
 	for _, file := range listFiles("extra") {
-		syncthingPkg.installationFiles = append(syncthingPkg.installationFiles, archiveFile{src: file, dst: "deb/usr/share/doc/syncthing/" + filepath.Base(file), perm: 0644})
+		syncthingPkg.installationFiles = append(syncthingPkg.installationFiles, archiveFile{src: file, dst: "deb/usr/share/doc/syncthing/" + filepath.Base(file), perm: 0o644})
 	}
 	targets["syncthing"] = syncthingPkg
 }
@@ -306,10 +288,10 @@ func runCommand(cmd string, target target) {
 		build(target, tags)
 
 	case "test":
-		test(strings.Fields(extraTags), "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
+		test(strings.Fields(extraTags), "github.com/syncthing/syncthing/internal/...", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
 
 	case "bench":
-		bench(strings.Fields(extraTags), "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
+		bench(strings.Fields(extraTags), "github.com/syncthing/syncthing/internal/...", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/...")
 
 	case "integration":
 		integration(false)
@@ -347,7 +329,7 @@ func runCommand(cmd string, target target) {
 		writeCompatJSON()
 
 	case "deb":
-		buildDeb(target)
+		buildDeb(target, tags)
 
 	case "vet":
 		metalintShort()
@@ -397,7 +379,6 @@ func parseFlags() {
 	flag.IntVar(&numVersions, "num-versions", numVersions, "Number of versions for changelog command")
 	flag.StringVar(&run, "run", "", "Specify which tests to run")
 	flag.StringVar(&benchRun, "bench", "", "Specify which benchmarks to run")
-	flag.BoolVar(&withNextGenGUI, "with-next-gen-gui", withNextGenGUI, "Also build 'newgui'")
 	flag.StringVar(&buildOut, "build-out", "", "Set the '-o' value for 'go build'")
 	flag.Parse()
 }
@@ -405,7 +386,6 @@ func parseFlags() {
 func test(tags []string, pkgs ...string) {
 	lazyRebuildAssets()
 
-	tags = append(tags, "purego")
 	args := []string{"test", "-tags", strings.Join(tags, " ")}
 	if long {
 		timeout = longTimeout
@@ -439,7 +419,7 @@ func bench(tags []string, pkgs ...string) {
 func integration(bench bool) {
 	lazyRebuildAssets()
 	args := []string{"test", "-v", "-timeout", "60m", "-tags"}
-	tags := "purego,integration"
+	tags := "integration"
 	if bench {
 		tags += ",benchmark"
 	}
@@ -471,10 +451,6 @@ func benchArgs() []string {
 }
 
 func install(target target, tags []string) {
-	if (target.name == "syncthing" || target.name == "") && !withNextGenGUI {
-		log.Println("Notice: Next generation GUI will not be built; see --with-next-gen-gui.")
-	}
-
 	lazyRebuildAssets()
 
 	tags = append(target.tags, tags...)
@@ -498,16 +474,12 @@ func install(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	args := []string{"install", "-v"}
+	args := []string{"install"}
 	args = appendParameters(args, tags, target.buildPkgs...)
 	runPrint(goCmd, args...)
 }
 
 func build(target target, tags []string) {
-	if (target.name == "syncthing" || target.name == "") && !withNextGenGUI {
-		log.Println("Notice: Next generation GUI will not be built; see --with-next-gen-gui.")
-	}
-
 	lazyRebuildAssets()
 	tags = append(target.tags, tags...)
 
@@ -530,7 +502,7 @@ func build(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	args := []string{"build", "-v"}
+	args := []string{"build"}
 	if buildOut != "" {
 		args = append(args, "-o", buildOut)
 	}
@@ -542,13 +514,6 @@ func setBuildEnvVars() {
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
 	os.Setenv("CC", cc)
-	if os.Getenv("CGO_ENABLED") == "" {
-		switch goos {
-		case "darwin", "solaris":
-		default:
-			os.Setenv("CGO_ENABLED", "0")
-		}
-	}
 }
 
 func appendParameters(args []string, tags []string, pkgs ...string) []string {
@@ -627,7 +592,7 @@ func buildZip(target target, tags []string) {
 	fmt.Println(filename)
 }
 
-func buildDeb(target target) {
+func buildDeb(target target, tags []string) {
 	os.RemoveAll("deb")
 
 	// "goarch" here is set to whatever the Debian packages expect. We correct
@@ -641,7 +606,7 @@ func buildDeb(target target) {
 		goarch = "arm"
 	}
 
-	build(target, []string{"noupgrade"})
+	build(target, append(tags, "noupgrade"))
 
 	for i := range target.installationFiles {
 		target.installationFiles[i].src = strings.Replace(target.installationFiles[i].src, "{{binary}}", target.BinaryName(), 1)
@@ -663,6 +628,9 @@ func buildDeb(target target) {
 		// than just 0.14.26. This rectifies that.
 		debver = strings.Replace(debver, "-", "~", -1)
 	}
+	if strings.Contains(debver, "_") {
+		debver = strings.Replace(debver, "_", "~", -1)
+	}
 	args := []string{
 		"-t", "deb",
 		"-s", "dir",
@@ -750,7 +718,7 @@ func shouldBuildSyso(dir string) (string, error) {
 	}
 
 	jsonPath := filepath.Join(dir, "versioninfo.json")
-	err = os.WriteFile(jsonPath, bs, 0644)
+	err = os.WriteFile(jsonPath, bs, 0o644)
 	if err != nil {
 		return "", errors.New("failed to create " + jsonPath + ": " + err.Error())
 	}
@@ -764,12 +732,9 @@ func shouldBuildSyso(dir string) (string, error) {
 	sysoPath := filepath.Join(dir, "cmd", "syncthing", "resource.syso")
 
 	// See https://github.com/josephspurrier/goversioninfo#command-line-flags
-	armOption := ""
-	if strings.Contains(goarch, "arm") {
-		armOption = "-arm=true"
-	}
-
-	if _, err := runError("goversioninfo", "-o", sysoPath, armOption); err != nil {
+	arm := strings.HasPrefix(goarch, "arm")
+	a64 := strings.Contains(goarch, "64")
+	if _, err := runError("goversioninfo", "-o", sysoPath, fmt.Sprintf("-arm=%v", arm), fmt.Sprintf("-64=%v", a64)); err != nil {
 		return "", errors.New("failed to create " + sysoPath + ": " + err.Error())
 	}
 
@@ -809,7 +774,7 @@ func copyFile(src, dst string, perm os.FileMode) error {
 	}
 
 copy:
-	os.MkdirAll(filepath.Dir(dst), 0777)
+	os.MkdirAll(filepath.Dir(dst), 0o777)
 	if err := os.WriteFile(dst, in, perm); err != nil {
 		return err
 	}
@@ -841,43 +806,11 @@ func lazyRebuildAssets() {
 	shouldRebuild := shouldRebuildAssets("lib/api/auto/gui.files.go", "gui") ||
 		shouldRebuildAssets("cmd/infra/strelaypoolsrv/auto/gui.files.go", "cmd/infra/strelaypoolsrv/gui")
 
-	if withNextGenGUI {
-		shouldRebuild = buildNextGenGUI() || shouldRebuild
-	}
-
 	if shouldRebuild {
 		rebuildAssets()
 	}
 }
 
-func buildNextGenGUI() bool {
-	// Check if we need to run the npm process, and if so also set the flag
-	// to rebuild Go assets afterwards. The index.html is regenerated every
-	// time by the build process. This assumes the new GUI ends up in
-	// next-gen-gui/dist/next-gen-gui.
-
-	if !shouldRebuildAssets("gui/next-gen-gui/index.html", "next-gen-gui") {
-		// The GUI is up to date.
-		return false
-	}
-
-	runPrintInDir("next-gen-gui", "npm", "install")
-	runPrintInDir("next-gen-gui", "npm", "run", "build", "--", "--prod", "--subresource-integrity")
-
-	rmr("gui/tech-ui")
-
-	for _, src := range listFiles("next-gen-gui/dist") {
-		rel, _ := filepath.Rel("next-gen-gui/dist", src)
-		dst := filepath.Join("gui", rel)
-		if err := copyFile(src, dst, 0644); err != nil {
-			fmt.Println("copy:", err)
-			os.Exit(1)
-		}
-	}
-
-	return true
-}
-
 func shouldRebuildAssets(target, srcdir string) bool {
 	info, err := os.Stat(target)
 	if err != nil {
@@ -925,22 +858,9 @@ func updateDependencies() {
 }
 
 func proto() {
-	pv := protobufVersion()
-	repo := "https://github.com/gogo/protobuf.git"
-	path := filepath.Join("repos", "protobuf")
-
-	runPrint(goCmd, "install", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
-	os.MkdirAll("repos", 0755)
-
-	if _, err := os.Stat(path); err != nil {
-		runPrint("git", "clone", repo, path)
-	} else {
-		runPrintInDir(path, "git", "fetch")
-	}
-	runPrintInDir(path, "git", "checkout", pv)
-
-	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/cmd/stdiscosrv")
-	runPrint(goCmd, "generate", "proto/generate.go")
+	// buf needs to be installed
+	// https://buf.build/docs/installation/
+	runPrint("buf", "generate")
 }
 
 func testmocks() {
@@ -950,7 +870,6 @@ func testmocks() {
 		"github.com/syncthing/syncthing/lib/connections",
 		"github.com/syncthing/syncthing/lib/discover",
 		"github.com/syncthing/syncthing/lib/events",
-		"github.com/syncthing/syncthing/lib/logger",
 		"github.com/syncthing/syncthing/lib/model",
 		"github.com/syncthing/syncthing/lib/protocol",
 	}
@@ -981,6 +900,7 @@ func weblate() {
 func ldflags(tags []string) string {
 	b := new(strings.Builder)
 	b.WriteString("-w")
+	b.WriteString(" -buildid=")
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Version=%s", version)
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp=%d", buildStamp())
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User=%s", buildUser())
@@ -1002,6 +922,9 @@ func rmr(paths ...string) {
 }
 
 func getReleaseVersion() (string, error) {
+	if ver := os.Getenv("VERSION"); ver != "" {
+		return strings.TrimSpace(ver), nil
+	}
 	bs, err := os.ReadFile("RELEASE")
 	if err != nil {
 		return "", err
@@ -1046,7 +969,7 @@ func getGitVersion() (string, error) {
 }
 
 func getVersion() string {
-	// First try for a RELEASE file,
+	// First try for a RELEASE file or $VERSION env var,
 	if ver, err := getReleaseVersion(); err == nil {
 		return ver
 	}
@@ -1375,10 +1298,7 @@ func zipFile(out string, files []archiveFile) {
 }
 
 func codesign(target target) {
-	switch goos {
-	case "windows":
-		windowsCodesign(target.BinaryName())
-	case "darwin":
+	if goos == "darwin" {
 		macosCodesign(target.BinaryName())
 	}
 }
@@ -1402,70 +1322,6 @@ func macosCodesign(file string) {
 	}
 }
 
-func windowsCodesign(file string) {
-	st := "signtool.exe"
-
-	if path := os.Getenv("CODESIGN_SIGNTOOL"); path != "" {
-		st = path
-	}
-
-	for i, algo := range []string{"sha1", "sha256"} {
-		args := []string{"sign", "/fd", algo}
-		if f := os.Getenv("CODESIGN_CERTIFICATE_FILE"); f != "" {
-			args = append(args, "/f", f)
-		} else if b := os.Getenv("CODESIGN_CERTIFICATE_BASE64"); b != "" {
-			// Decode the PFX certificate from base64.
-			bs, err := base64.RawStdEncoding.DecodeString(b)
-			if err != nil {
-				log.Println("Codesign: signing failed: decoding base64:", err)
-				return
-			}
-
-			// Write it to a temporary file
-			f, err := os.CreateTemp("", "codesign-*.pfx")
-			if err != nil {
-				log.Println("Codesign: signing failed: creating temp file:", err)
-				return
-			}
-			_ = f.Chmod(0600) // best effort remove other users' access
-			defer os.Remove(f.Name())
-			if _, err := f.Write(bs); err != nil {
-				log.Println("Codesign: signing failed: writing temp file:", err)
-				return
-			}
-			if err := f.Close(); err != nil {
-				log.Println("Codesign: signing failed: closing temp file:", err)
-				return
-			}
-
-			// Use that when signing
-			args = append(args, "/f", f.Name())
-		}
-		if p := os.Getenv("CODESIGN_CERTIFICATE_PASSWORD"); p != "" {
-			args = append(args, "/p", p)
-		}
-		if tr := os.Getenv("CODESIGN_TIMESTAMP_SERVER"); tr != "" {
-			switch algo {
-			case "sha256":
-				args = append(args, "/tr", tr, "/td", algo)
-			default:
-				args = append(args, "/t", tr)
-			}
-		}
-		if i > 0 {
-			args = append(args, "/as")
-		}
-		args = append(args, file)
-
-		bs, err := runError(st, args...)
-		if err != nil {
-			log.Printf("Codesign: signing failed: %v: %s", err, string(bs))
-			return
-		}
-		log.Println("Codesign: successfully signed", file, "using", algo)
-	}
-}
-
 func metalint() {
 	lazyRebuildAssets()
 	runPrint(goCmd, "test", "-run", "Metalint", "./meta")
@@ -1483,14 +1339,6 @@ func (t target) BinaryName() string {
 	return t.binaryName
 }
 
-func protobufVersion() string {
-	bs, err := runError(goCmd, "list", "-f", "{{.Version}}", "-m", "github.com/gogo/protobuf")
-	if err != nil {
-		log.Fatal("Getting protobuf version:", err)
-	}
-	return string(bs)
-}
-
 func currentAndLatestVersions(n int) ([]string, error) {
 	bs, err := runError("git", "tag", "--sort", "taggerdate")
 	if err != nil {

build.sh

@@ -23,10 +23,11 @@ case "${1:-default}" in
 
 	prerelease)
 		script authors
+		script copyrights
 		build weblate
 		pushd man ; ./refresh.sh ; popd
 		git add -A gui man AUTHORS
-		git commit -m 'gui, man, authors: Update docs, translations, and contributors'
+		git commit -m 'chore(gui, man, authors): update docs, translations, and contributors'
 		;;
 
 	*)

cmd/dev/stdisco/main.go

@@ -15,6 +15,9 @@ import (
 	"strings"
 	"time"
 
+	"google.golang.org/protobuf/proto"
+
+	"github.com/syncthing/syncthing/internal/gen/discoproto"
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/beacon"
 	"github.com/syncthing/syncthing/lib/discover"
@@ -75,20 +78,21 @@ func recv(bc beacon.Interface) {
 			continue
 		}
 
-		var ann discover.Announce
-		ann.Unmarshal(data[4:])
+		var ann discoproto.Announce
+		proto.Unmarshal(data[4:], &ann)
 
-		if ann.ID == myID {
+		id, _ := protocol.DeviceIDFromBytes(ann.Id)
+		if id == myID {
 			// This is one of our own fake packets, don't print it.
 			continue
 		}
 
 		// Print announcement details for the first packet from a given
 		// device ID and source address, or if -all was given.
-		key := ann.ID.String() + src.String()
+		key := id.String() + src.String()
 		if all || !seen[key] {
 			log.Printf("Announcement from %v\n", src)
-			log.Printf(" %v at %s\n", ann.ID, strings.Join(ann.Addresses, ", "))
+			log.Printf(" %v at %s\n", id, strings.Join(ann.Addresses, ", "))
 			seen[key] = true
 		}
 	}
@@ -96,11 +100,11 @@ func recv(bc beacon.Interface) {
 
 // sends fake discovery announcements once every second
 func send(bc beacon.Interface) {
-	ann := discover.Announce{
-		ID:        myID,
+	ann := &discoproto.Announce{
+		Id:        myID[:],
 		Addresses: []string{"tcp://fake.example.com:12345"},
 	}
-	bs, _ := ann.Marshal()
+	bs, _ := proto.Marshal(ann)
 
 	for {
 		bc.Send(bs)

cmd/dev/stfileinfo/main.go

@@ -72,7 +72,7 @@ func main() {
 		if *standardBlocks || blockSize < protocol.MinBlockSize {
 			blockSize = protocol.BlockSize(fi.Size())
 		}
-		bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
+		bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/infra/stcrashreceiver/diskstore.go

@@ -8,6 +8,7 @@ package main
 
 import (
 	"bytes"
+	"cmp"
 	"compress/gzip"
 	"context"
 	"io"
@@ -15,7 +16,7 @@ import (
 	"math"
 	"os"
 	"path/filepath"
-	"sort"
+	"slices"
 	"time"
 )
 
@@ -177,8 +178,8 @@ func (d *diskStore) inventory() error {
 		})
 		return nil
 	})
-	sort.Slice(d.currentFiles, func(i, j int) bool {
-		return d.currentFiles[i].mtime < d.currentFiles[j].mtime
+	slices.SortFunc(d.currentFiles, func(a, b currentFile) int {
+		return cmp.Compare(a.mtime, b.mtime)
 	})
 	var oldest time.Duration
 	if len(d.currentFiles) > 0 {

cmd/infra/stcrashreceiver/main.go

@@ -44,7 +44,7 @@ type cli struct {
 	SentryQueue    int    `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"`
 	DiskQueue      int    `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"`
 	MetricsListen  string `help:"HTTP listen address for metrics" default:":8081" env:"METRICS_LISTEN_ADDRESS"`
-	IngorePatterns string `help:"File containing ignore patterns (regexp)" env:"IGNORE_PATTERNS" type:"existingfile"`
+	IgnorePatterns string `help:"File containing ignore patterns (regexp)" env:"IGNORE_PATTERNS" type:"existingfile"`
 }
 
 func main() {
@@ -68,9 +68,9 @@ func main() {
 	go ss.Serve(context.Background())
 
 	var ip *ignorePatterns
-	if params.IngorePatterns != "" {
+	if params.IgnorePatterns != "" {
 		var err error
-		ip, err = loadIgnorePatterns(params.IngorePatterns)
+		ip, err = loadIgnorePatterns(params.IgnorePatterns)
 		if err != nil {
 			log.Fatalf("Failed to load ignore patterns: %v", err)
 		}
@@ -118,7 +118,7 @@ func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http
 		bs, err := io.ReadAll(lr)
 		req.Body.Close()
 		if err != nil {
-			http.Error(w, err.Error(), 500)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
 
@@ -130,7 +130,7 @@ func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http
 		var reports []ur.FailureReport
 		err = json.Unmarshal(bs, &reports)
 		if err != nil {
-			http.Error(w, err.Error(), 400)
+			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
 		if len(reports) == 0 {
@@ -141,7 +141,7 @@ func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http
 
 		version, err := build.ParseVersion(reports[0].Version)
 		if err != nil {
-			http.Error(w, err.Error(), 400)
+			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
 		for _, r := range reports {

cmd/infra/stcrashreceiver/sentry.go

@@ -90,7 +90,7 @@ func sendReport(dsn string, pkt *raven.Packet, userID string) error {
 	}
 
 	// The client sets release and such on the packet before sending, in the
-	// misguided idea that it knows this better than than the packet we give
+	// misguided idea that it knows this better than the packet we give
 	// it. So we copy the values from the packet to the client first...
 	cli.SetRelease(pkt.Release)
 	cli.SetEnvironment(pkt.Environment)
@@ -136,7 +136,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
 
 	r := bytes.NewReader(report)
 	ctx, _, err := stack.ScanSnapshot(r, io.Discard, stack.DefaultOpts())
-	if err != nil && err != io.EOF {
+	if err != nil && !errors.Is(err, io.EOF) {
 		return nil, err
 	}
 	if ctx == nil || len(ctx.Goroutines) == 0 {

cmd/infra/stcrashreceiver/sourcecodeloader.go

@@ -71,7 +71,6 @@ func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]b
 
 		url := urlPrefix + l.version + filename[idx:]
 		resp, err := l.client.Get(url)
-
 		if err != nil {
 			fmt.Println("Loading source:", err)
 			return nil, 0

cmd/infra/stcrashreceiver/util.go

@@ -10,7 +10,7 @@ import (
 	"bytes"
 	"compress/gzip"
 	"crypto/sha256"
-	"fmt"
+	"encoding/hex"
 	"net"
 	"net/http"
 	"os"
@@ -23,7 +23,7 @@ import (
 // remote IP, and the current month.
 func userIDFor(req *http.Request) string {
 	addr := req.RemoteAddr
-	if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
+	if fwd := req.Header.Get("X-Forwarded-For"); fwd != "" {
 		addr = fwd
 	}
 	if host, _, err := net.SplitHostPort(addr); err == nil {
@@ -32,7 +32,7 @@ func userIDFor(req *http.Request) string {
 	now := time.Now().Format("200601")
 	salt := "stcrashreporter"
 	hash := sha256.Sum256([]byte(salt + addr + now))
-	return fmt.Sprintf("%x", hash[:8])
+	return hex.EncodeToString(hash[:8])
 }
 
 // 01234567890abcdef... => 01/23
@@ -52,5 +52,5 @@ func compressAndWrite(bs []byte, fullPath string) error {
 	gw.Close()
 
 	// Create an output file with the compressed report
-	return os.WriteFile(fullPath, buf.Bytes(), 0644)
+	return os.WriteFile(fullPath, buf.Bytes(), 0o644)
 }

cmd/infra/strelaypoolsrv/README.md

@@ -10,7 +10,7 @@ to NAT or firewall issues.
 
 There is very little reason why you'd want to run this yourself, as
 `relaypoolsrv` is just used for announcement and lookup of public relay
-servers. If you are looking to setup a private or a public relay, please
+servers. If you are looking to set up a private or a public relay, please
 check the documentation for
 [relaysrv](https://github.com/syncthing/relaysrv), which also explains how
 to join the default public pool.
@@ -21,4 +21,3 @@ See `relaypoolsrv -help` for configuration options.
 
 [oschwald/geoip2-golang](https://github.com/oschwald/geoip2-golang), [oschwald/maxminddb-golang](https://github.com/oschwald/maxminddb-golang), Copyright (C) 2015 [Gregory J. Oschwald](mailto:oschwald@gmail.com).
 
-[lib/pq](https://github.com/lib/pq)</a>, Copyright (C) 2011-2013 'pq' Contributors Portions Copyright (C) 2011 Blake Mizerany.

cmd/infra/strelaypoolsrv/main.go

@@ -17,19 +17,21 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
 	lru "github.com/hashicorp/golang-lru/v2"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+
 	"github.com/syncthing/syncthing/cmd/infra/strelaypoolsrv/auto"
 	"github.com/syncthing/syncthing/lib/assets"
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/geoip"
 	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/relay/client"
-	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 )
 
@@ -109,10 +111,11 @@ var (
 	requestProcessors = 8
 	geoipLicenseKey   = os.Getenv("GEOIP_LICENSE_KEY")
 	geoipAccountID, _ = strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID"))
+	maxRelaysReturned = 100
 
 	requests chan request
 
-	mut             = sync.NewRWMutex()
+	mut             sync.RWMutex
 	knownRelays     = make([]*relay, 0)
 	permanentRelays = make([]*relay, 0)
 	evictionTimers  = make(map[string]*time.Timer)
@@ -140,6 +143,7 @@ func main() {
 	flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
 	flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
 	flag.StringVar(&geoipLicenseKey, "geoip-license-key", geoipLicenseKey, "License key for GeoIP database")
+	flag.IntVar(&maxRelaysReturned, "max-relays-returned", maxRelaysReturned, "Maximum number of relays returned for a normal endpoint query")
 
 	flag.Parse()
 
@@ -158,7 +162,7 @@ func main() {
 
 	testCert = createTestCertificate()
 
-	for i := 0; i < requestProcessors; i++ {
+	for range requestProcessors {
 		go requestProcessor(geoip)
 	}
 
@@ -176,7 +180,7 @@ func main() {
 				relayTestsTotal.WithLabelValues("success").Inc()
 			}
 		}
-		// Run the the stats refresher once the relays are loaded.
+		// Run the stats refresher once the relays are loaded.
 		statsRefresher(statsRefresh)
 	}()
 
@@ -330,6 +334,10 @@ func handleEndpointShort(rw http.ResponseWriter, r *http.Request) {
 		relays = append(relays, relayShort{URL: slimURL(r.URL)})
 	}
 	mut.RUnlock()
+	if len(relays) > maxRelaysReturned {
+		rand.Shuffle(relays)
+		relays = relays[:maxRelaysReturned]
+	}
 
 	_ = json.NewEncoder(rw).Encode(map[string][]relayShort{
 		"relays": relays,
@@ -612,7 +620,7 @@ func createTestCertificate() tls.Certificate {
 	}
 
 	certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
-	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 20*365)
+	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 20*365, false)
 	if err != nil {
 		log.Fatalln("Failed to create test X509 key pair:", err)
 	}
@@ -645,6 +653,7 @@ func getLocation(host string, geoip *geoip.Provider) location {
 
 type loggingResponseWriter struct {
 	http.ResponseWriter
+
 	statusCode int
 }
 

cmd/infra/strelaypoolsrv/main_test.go

@@ -13,7 +13,6 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"strings"
-	"sync"
 	"testing"
 )
 
@@ -28,8 +27,6 @@ func init() {
 		{URL: "known2"},
 		{URL: "known3"},
 	}
-
-	mut = new(sync.RWMutex)
 }
 
 // Regression test: handleGetRequest should not modify permanentRelays.

cmd/infra/strelaypoolsrv/stats.go

@@ -6,10 +6,10 @@ import (
 	"encoding/json"
 	"net"
 	"net/http"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/syncthing/syncthing/lib/sync"
 )
 
 var (
@@ -104,7 +104,7 @@ func refreshStats() {
 	mut.RUnlock()
 
 	now := time.Now()
-	wg := sync.NewWaitGroup()
+	var wg sync.WaitGroup
 
 	results := make(chan statsFetchResult, len(relays))
 	for _, rel := range relays {
@@ -173,7 +173,7 @@ func fetchStats(relay *relay) *stats {
 
 	var stats stats
 
-	if json.NewDecoder(response.Body).Decode(&stats); err != nil {
+	if err := json.NewDecoder(response.Body).Decode(&stats); err != nil {
 		return nil
 	}
 	return &stats

cmd/infra/stupgrades/main.go

@@ -7,19 +7,24 @@
 package main
 
 import (
-	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
-	"log"
+	"log/slog"
+	"net"
 	"net/http"
 	"os"
+	"regexp"
 	"sort"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/alecthomas/kong"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/syncthing/syncthing/internal/slogutil"
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/httpcache"
 	"github.com/syncthing/syncthing/lib/upgrade"
@@ -27,7 +32,7 @@ import (
 
 type cli struct {
 	Listen        string        `default:":8080" help:"Listen address"`
-	MetricsListen string        `default:":8081" help:"Listen address for metrics"`
+	MetricsListen string        `default:":8082" help:"Listen address for metrics"`
 	URL           string        `short:"u" default:"https://api.github.com/repos/syncthing/syncthing/releases?per_page=25" help:"GitHub releases url"`
 	Forward       []string      `short:"f" help:"Forwarded pages, format: /path->https://example/com/url"`
 	CacheTime     time.Duration `default:"15m" help:"Cache time"`
@@ -36,6 +41,10 @@ type cli struct {
 func main() {
 	var params cli
 	kong.Parse(&params)
+	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
+		Level: slog.LevelInfo,
+	})))
+
 	if err := server(&params); err != nil {
 		fmt.Printf("Error: %v\n", err)
 		os.Exit(1)
@@ -46,24 +55,47 @@ func server(params *cli) error {
 	if params.MetricsListen != "" {
 		mux := http.NewServeMux()
 		mux.Handle("/metrics", promhttp.Handler())
+		metricsListen, err := net.Listen("tcp", params.MetricsListen)
+		if err != nil {
+			return fmt.Errorf("metrics: %w", err)
+		}
+		slog.Info("Metrics listener started", slogutil.Address(params.MetricsListen))
 		go func() {
-			log.Println("Listening for metrics on", params.MetricsListen)
-			if err := http.ListenAndServe(params.MetricsListen, mux); err != nil {
-				log.Fatalf("Failed to start metrics server: %v", err)
+			if err := http.Serve(metricsListen, mux); err != nil {
+				slog.Warn("Metrics server returned", slogutil.Error(err))
 			}
 		}()
 	}
 
+	cache := &cachedReleases{url: params.URL}
+	if err := cache.Update(context.Background()); err != nil {
+		return fmt.Errorf("initial cache update: %w", err)
+	} else {
+		slog.Info("Initial cache update done")
+	}
+
+	go func() {
+		for range time.NewTicker(params.CacheTime).C {
+			slog.Info("Refreshing cached releases", slogutil.URI(params.URL))
+			if err := cache.Update(context.Background()); err != nil {
+				slog.Error("Failed to refresh cached releases", slogutil.URI(params.URL), slogutil.Error(err))
+			}
+		}
+	}()
+
+	ghRels := &githubReleases{cache: cache}
 	mux := http.NewServeMux()
-	mux.Handle("/meta.json", httpcache.SinglePath(&githubReleases{url: params.URL}, params.CacheTime))
+	mux.HandleFunc("/ping", ghRels.servePing)
+	mux.HandleFunc("/meta.json", ghRels.serveReleases)
 
 	for _, fwd := range params.Forward {
 		path, url, ok := strings.Cut(fwd, "->")
 		if !ok {
 			return fmt.Errorf("invalid forward: %q", fwd)
 		}
-		log.Println("Forwarding", path, "to", url)
-		mux.Handle(path, httpcache.SinglePath(&proxy{url: url}, params.CacheTime))
+		slog.Info("Forwarding", "from", path, "to", url)
+		name := strings.ReplaceAll(path, "/", "_")
+		mux.Handle(path, httpcache.SinglePath(&proxy{name: name, url: url}, params.CacheTime))
 	}
 
 	srv := &http.Server{
@@ -73,60 +105,76 @@ func server(params *cli) error {
 		WriteTimeout: 10 * time.Second,
 	}
 	srv.SetKeepAlivesEnabled(false)
-	return srv.ListenAndServe()
+
+	srvListener, err := net.Listen("tcp", params.Listen)
+	if err != nil {
+		return fmt.Errorf("listen: %w", err)
+	}
+	slog.Info("Main listener started", slogutil.Address(params.Listen))
+
+	return srv.Serve(srvListener)
 }
 
 type githubReleases struct {
-	url string
+	cache *cachedReleases
 }
 
-func (p *githubReleases) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
-	log.Println("Fetching", p.url)
-	rels := upgrade.FetchLatestReleases(p.url, "")
-	if rels == nil {
-		http.Error(w, "no releases", http.StatusInternalServerError)
+func (p *githubReleases) servePing(w http.ResponseWriter, req *http.Request) {
+	rels := p.cache.Releases()
+
+	if len(rels) == 0 {
+		http.Error(w, "No releases available", http.StatusServiceUnavailable)
 		return
 	}
 
-	sort.Sort(upgrade.SortByRelease(rels))
-	rels = filterForLatest(rels)
+	w.Header().Set("Syncthing-Num-Releases", strconv.Itoa(len(rels)))
+	w.WriteHeader(http.StatusOK)
+}
 
-	// Move the URL used for browser downloads to the URL field, and remove
-	// the browser URL field. This avoids going via the GitHub API for
-	// downloads, since Syncthing uses the URL field.
-	for _, rel := range rels {
-		for j, asset := range rel.Assets {
-			rel.Assets[j].URL = asset.BrowserURL
-			rel.Assets[j].BrowserURL = ""
-		}
+func (p *githubReleases) serveReleases(w http.ResponseWriter, req *http.Request) {
+	rels := p.cache.Releases()
+
+	ua := req.Header.Get("User-Agent")
+	osv := req.Header.Get("Syncthing-Os-Version")
+	if ua != "" && osv != "" {
+		// We should determine the compatibility of the releases.
+		rels = filterForCompatibility(rels, ua, osv)
+	} else {
+		metricFilterCalls.WithLabelValues("no-ua-or-osversion").Inc()
 	}
 
-	buf := new(bytes.Buffer)
-	_ = json.NewEncoder(buf).Encode(rels)
+	rels = filterForLatest(rels)
 
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	w.Header().Set("Access-Control-Allow-Origin", "*")
 	w.Header().Set("Access-Control-Allow-Methods", "GET")
-	w.Write(buf.Bytes())
+	w.Header().Set("Cache-Control", "public, max-age=900")
+	w.Header().Set("Vary", "User-Agent, Syncthing-Os-Version")
+	_ = json.NewEncoder(w).Encode(rels)
+
+	metricUpgradeChecks.Inc()
 }
 
 type proxy struct {
-	url string
+	name string
+	url  string
 }
 
 func (p *proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	log.Println("Fetching", p.url)
 	req, err := http.NewRequestWithContext(req.Context(), http.MethodGet, p.url, nil)
 	if err != nil {
+		metricHTTPRequests.WithLabelValues(p.name, "error").Inc()
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
+		metricHTTPRequests.WithLabelValues(p.name, "error").Inc()
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 	defer resp.Body.Close()
+	metricHTTPRequests.WithLabelValues(p.name, "success").Inc()
 
 	ct := resp.Header.Get("Content-Type")
 	w.Header().Set("Content-Type", ct)
@@ -154,18 +202,174 @@ func (p *proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 // looking for a prerelease at all.
 func filterForLatest(rels []upgrade.Release) []upgrade.Release {
 	var filtered []upgrade.Release
-	var havePre bool
+	havePre := make(map[string]bool)
+	haveStable := make(map[string]bool)
 	for _, rel := range rels {
-		if !rel.Prerelease {
-			// We found a stable version, we're good now.
+		major, _, _ := strings.Cut(rel.Tag, ".")
+		if !rel.Prerelease && !haveStable[major] {
+			// Remember the first non-pre for each major
 			filtered = append(filtered, rel)
-			break
+			haveStable[major] = true
+			continue
 		}
-		if rel.Prerelease && !havePre {
-			// We remember the first prerelease we find.
+		if rel.Prerelease && !havePre[major] && !haveStable[major] {
+			// We remember the first prerelease we find, unless we've
+			// already found a non-pre of the same major.
 			filtered = append(filtered, rel)
-			havePre = true
+			havePre[major] = true
 		}
 	}
 	return filtered
 }
+
+var userAgentOSArchExp = regexp.MustCompile(`^syncthing.*\(.+ (\w+)-(\w+)\)$`)
+
+func filterForCompatibility(rels []upgrade.Release, ua, osv string) []upgrade.Release {
+	osArch := userAgentOSArchExp.FindStringSubmatch(ua)
+	if len(osArch) != 3 {
+		metricFilterCalls.WithLabelValues("bad-os-arch").Inc()
+		return rels
+	}
+	os := osArch[1]
+
+	var filtered []upgrade.Release
+	for _, rel := range rels {
+		if rel.Compatibility == nil {
+			// No requirements means it's compatible with everything.
+			filtered = append(filtered, rel)
+			continue
+		}
+
+		req, ok := rel.Compatibility.Requirements[os]
+		if !ok {
+			// No entry for the current OS means it's compatible.
+			filtered = append(filtered, rel)
+			continue
+		}
+
+		if upgrade.CompareVersions(osv, req) >= 0 {
+			filtered = append(filtered, rel)
+			continue
+		}
+	}
+
+	if len(filtered) != len(rels) {
+		metricFilterCalls.WithLabelValues("filtered").Inc()
+	} else {
+		metricFilterCalls.WithLabelValues("unchanged").Inc()
+	}
+
+	return filtered
+}
+
+type cachedReleases struct {
+	url                  string
+	mut                  sync.RWMutex
+	current              []upgrade.Release
+	latestRel, latestPre string
+}
+
+func (c *cachedReleases) Releases() []upgrade.Release {
+	c.mut.RLock()
+	defer c.mut.RUnlock()
+	return c.current
+}
+
+func (c *cachedReleases) Update(ctx context.Context) error {
+	rels, err := fetchGithubReleases(ctx, c.url)
+	if err != nil {
+		return err
+	}
+	latestRel, latestPre := "", ""
+	for _, rel := range rels {
+		if !rel.Prerelease && latestRel == "" {
+			latestRel = rel.Tag
+		}
+		if rel.Prerelease && latestPre == "" {
+			latestPre = rel.Tag
+		}
+		if latestRel != "" && latestPre != "" {
+			break
+		}
+	}
+	c.mut.Lock()
+	c.current = rels
+	if latestRel != c.latestRel || latestPre != c.latestPre {
+		metricLatestReleaseInfo.DeleteLabelValues(c.latestRel, c.latestPre)
+		metricLatestReleaseInfo.WithLabelValues(latestRel, latestPre).Set(1)
+		c.latestRel = latestRel
+		c.latestPre = latestPre
+	}
+	c.mut.Unlock()
+	return nil
+}
+
+func fetchGithubReleases(ctx context.Context, url string) ([]upgrade.Release, error) {
+	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, nil)
+	if err != nil {
+		metricHTTPRequests.WithLabelValues("github-releases", "error").Inc()
+		return nil, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		metricHTTPRequests.WithLabelValues("github-releases", "error").Inc()
+		return nil, err
+	}
+	defer resp.Body.Close()
+	var rels []upgrade.Release
+	if err := json.NewDecoder(resp.Body).Decode(&rels); err != nil {
+		metricHTTPRequests.WithLabelValues("github-releases", "error").Inc()
+		return nil, err
+	}
+	metricHTTPRequests.WithLabelValues("github-releases", "success").Inc()
+
+	// Move the URL used for browser downloads to the URL field, and remove
+	// the browser URL field. This avoids going via the GitHub API for
+	// downloads, since Syncthing uses the URL field.
+	for _, rel := range rels {
+		for j, asset := range rel.Assets {
+			rel.Assets[j].URL = asset.BrowserURL
+			rel.Assets[j].BrowserURL = ""
+		}
+	}
+
+	addReleaseCompatibility(ctx, rels)
+
+	sort.Sort(upgrade.SortByRelease(rels))
+	return rels, nil
+}
+
+func addReleaseCompatibility(ctx context.Context, rels []upgrade.Release) {
+	for i := range rels {
+		rel := &rels[i]
+		for i, asset := range rel.Assets {
+			if asset.Name != "compat.json" {
+				continue
+			}
+
+			// Load compat.json into the Compatibility field
+			req, err := http.NewRequestWithContext(ctx, http.MethodGet, asset.URL, nil)
+			if err != nil {
+				metricHTTPRequests.WithLabelValues("compat-json", "error").Inc()
+				break
+			}
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				metricHTTPRequests.WithLabelValues("compat-json", "error").Inc()
+				break
+			}
+			if resp.StatusCode != http.StatusOK {
+				metricHTTPRequests.WithLabelValues("compat-json", "error").Inc()
+				resp.Body.Close()
+				break
+			}
+			_ = json.NewDecoder(io.LimitReader(resp.Body, 10<<10)).Decode(&rel.Compatibility)
+			metricHTTPRequests.WithLabelValues("compat-json", "success").Inc()
+			resp.Body.Close()
+
+			// Remove compat.json from the asset list since it's been processed
+			rel.Assets = append(rel.Assets[:i], rel.Assets[i+1:]...)
+			break
+		}
+	}
+}

cmd/infra/stupgrades/metrics.go

@@ -0,0 +1,36 @@
+// Copyright (C) 2024 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	metricUpgradeChecks = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "upgrade",
+		Name:      "metadata_requests",
+	})
+	metricFilterCalls = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "upgrade",
+		Name:      "filter_calls",
+	}, []string{"result"})
+	metricHTTPRequests = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "upgrade",
+		Name:      "http_requests",
+	}, []string{"target", "result"})
+	metricLatestReleaseInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "syncthing",
+		Subsystem: "upgrade",
+		Name:      "latest_release_info",
+		Help:      "Release information",
+	}, []string{"latest_release", "latest_pre"})
+)

cmd/infra/ursrv/aggregate/aggregate.go

@@ -1,226 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package aggregate
-
-import (
-	"database/sql"
-	"fmt"
-	"log"
-	"os"
-	"time"
-
-	_ "github.com/lib/pq"
-)
-
-type CLI struct {
-	DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"`
-}
-
-func (cli *CLI) Run() error {
-	log.SetFlags(log.Ltime | log.Ldate)
-	log.SetOutput(os.Stdout)
-
-	db, err := sql.Open("postgres", cli.DBConn)
-	if err != nil {
-		return fmt.Errorf("database: %w", err)
-	}
-	err = setupDB(db)
-	if err != nil {
-		return fmt.Errorf("database: %w", err)
-	}
-
-	for {
-		runAggregation(db)
-		// Sleep until one minute past next midnight
-		sleepUntilNext(24*time.Hour, 1*time.Minute)
-	}
-}
-
-func runAggregation(db *sql.DB) {
-	since := maxIndexedDay(db, "VersionSummary")
-	log.Println("Aggregating VersionSummary data since", since)
-	rows, err := aggregateVersionSummary(db, since.Add(24*time.Hour))
-	if err != nil {
-		log.Println("aggregate:", err)
-	}
-	log.Println("Inserted", rows, "rows")
-
-	since = maxIndexedDay(db, "Performance")
-	log.Println("Aggregating Performance data since", since)
-	rows, err = aggregatePerformance(db, since.Add(24*time.Hour))
-	if err != nil {
-		log.Println("aggregate:", err)
-	}
-	log.Println("Inserted", rows, "rows")
-
-	since = maxIndexedDay(db, "BlockStats")
-	log.Println("Aggregating BlockStats data since", since)
-	rows, err = aggregateBlockStats(db, since.Add(24*time.Hour))
-	if err != nil {
-		log.Println("aggregate:", err)
-	}
-	log.Println("Inserted", rows, "rows")
-}
-
-func sleepUntilNext(intv, margin time.Duration) {
-	now := time.Now().UTC()
-	next := now.Truncate(intv).Add(intv).Add(margin)
-	log.Println("Sleeping until", next)
-	time.Sleep(next.Sub(now))
-}
-
-func setupDB(db *sql.DB) error {
-	_, err := db.Exec(`CREATE TABLE IF NOT EXISTS VersionSummary (
-		Day TIMESTAMP NOT NULL,
-		Version VARCHAR(8) NOT NULL,
-		Count INTEGER NOT NULL
-	)`)
-	if err != nil {
-		return err
-	}
-
-	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Performance (
-		Day TIMESTAMP NOT NULL,
-		TotFiles INTEGER NOT NULL,
-		TotMiB INTEGER NOT NULL,
-		SHA256Perf DOUBLE PRECISION NOT NULL,
-		MemorySize INTEGER NOT NULL,
-		MemoryUsageMiB INTEGER NOT NULL
-	)`)
-	if err != nil {
-		return err
-	}
-
-	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS BlockStats (
-		Day TIMESTAMP NOT NULL,
-		Reports INTEGER NOT NULL,
-		Total BIGINT NOT NULL,
-		Renamed BIGINT NOT NULL,
-		Reused BIGINT NOT NULL,
-		Pulled BIGINT NOT NULL,
-		CopyOrigin BIGINT NOT NULL,
-		CopyOriginShifted BIGINT NOT NULL,
-		CopyElsewhere BIGINT NOT NULL
-	)`)
-	if err != nil {
-		return err
-	}
-
-	var t string
-
-	row := db.QueryRow(`SELECT 'UniqueDayVersionIndex'::regclass`)
-	if err := row.Scan(&t); err != nil {
-		_, _ = db.Exec(`CREATE UNIQUE INDEX UniqueDayVersionIndex ON VersionSummary (Day, Version)`)
-	}
-
-	row = db.QueryRow(`SELECT 'VersionDayIndex'::regclass`)
-	if err := row.Scan(&t); err != nil {
-		_, _ = db.Exec(`CREATE INDEX VersionDayIndex ON VersionSummary (Day)`)
-	}
-
-	row = db.QueryRow(`SELECT 'PerformanceDayIndex'::regclass`)
-	if err := row.Scan(&t); err != nil {
-		_, _ = db.Exec(`CREATE INDEX PerformanceDayIndex ON Performance (Day)`)
-	}
-
-	row = db.QueryRow(`SELECT 'BlockStatsDayIndex'::regclass`)
-	if err := row.Scan(&t); err != nil {
-		_, _ = db.Exec(`CREATE INDEX BlockStatsDayIndex ON BlockStats (Day)`)
-	}
-
-	return nil
-}
-
-func maxIndexedDay(db *sql.DB, table string) time.Time {
-	var t time.Time
-	row := db.QueryRow("SELECT MAX(DATE_TRUNC('day', Day)) FROM " + table)
-	err := row.Scan(&t)
-	if err != nil {
-		return time.Time{}
-	}
-	return t
-}
-
-func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) {
-	res, err := db.Exec(`INSERT INTO VersionSummary (
-	SELECT
-		DATE_TRUNC('day', Received) AS Day,
-		SUBSTRING(Report->>'version' FROM '^v\d.\d+') AS Ver,
-		COUNT(*) AS Count
-		FROM ReportsJson
-		WHERE
-			Received > $1
-			AND Received < DATE_TRUNC('day', NOW())
-			AND Report->>'version' like 'v_.%'
-		GROUP BY Day, Ver
-		);
-	`, since)
-	if err != nil {
-		return 0, err
-	}
-
-	return res.RowsAffected()
-}
-
-func aggregatePerformance(db *sql.DB, since time.Time) (int64, error) {
-	res, err := db.Exec(`INSERT INTO Performance (
-	SELECT
-		DATE_TRUNC('day', Received) AS Day,
-		AVG((Report->>'totFiles')::numeric) As TotFiles,
-		AVG((Report->>'totMiB')::numeric) As TotMiB,
-		AVG((Report->>'sha256Perf')::numeric) As SHA256Perf,
-		AVG((Report->>'memorySize')::numeric) As MemorySize,
-		AVG((Report->>'memoryUsageMiB')::numeric) As MemoryUsageMiB
-		FROM ReportsJson
-		WHERE
-			Received > $1
-			AND Received < DATE_TRUNC('day', NOW())
-			AND Report->>'version' like 'v_.%'
-			/* Some custom implementation reported bytes when we expect megabytes, cap at petabyte */
-			AND (Report->>'memorySize')::numeric < 1073741824
-		GROUP BY Day
-		);
-	`, since)
-	if err != nil {
-		return 0, err
-	}
-
-	return res.RowsAffected()
-}
-
-func aggregateBlockStats(db *sql.DB, since time.Time) (int64, error) {
-	// Filter out anything prior 0.14.41 as that has sum aggregations which
-	// made no sense.
-	res, err := db.Exec(`INSERT INTO BlockStats (
-	SELECT
-		DATE_TRUNC('day', Received) AS Day,
-		COUNT(1) As Reports,
-		SUM((Report->'blockStats'->>'total')::numeric)::bigint AS Total,
-		SUM((Report->'blockStats'->>'renamed')::numeric)::bigint AS Renamed,
-		SUM((Report->'blockStats'->>'reused')::numeric)::bigint AS Reused,
-		SUM((Report->'blockStats'->>'pulled')::numeric)::bigint AS Pulled,
-		SUM((Report->'blockStats'->>'copyOrigin')::numeric)::bigint AS CopyOrigin,
-		SUM((Report->'blockStats'->>'copyOriginShifted')::numeric)::bigint AS CopyOriginShifted,
-		SUM((Report->'blockStats'->>'copyElsewhere')::numeric)::bigint AS CopyElsewhere
-		FROM ReportsJson
-		WHERE
-			Received > $1
-			AND Received < DATE_TRUNC('day', NOW())
-			AND (Report->>'urVersion')::numeric >= 3
-			AND Report->>'version' like 'v_.%'
-			AND Report->>'version' NOT LIKE 'v0.14.40%'
-			AND Report->>'version' NOT LIKE 'v0.14.39%'
-			AND Report->>'version' NOT LIKE 'v0.14.38%'
-		GROUP BY Day
-	);
-	`, since)
-	if err != nil {
-		return 0, err
-	}
-
-	return res.RowsAffected()
-}

cmd/infra/ursrv/main.go

@@ -8,22 +8,22 @@ package main
 
 import (
 	"log"
+	"log/slog"
 	"os"
 
 	"github.com/alecthomas/kong"
-	"github.com/syncthing/syncthing/cmd/infra/ursrv/aggregate"
 	"github.com/syncthing/syncthing/cmd/infra/ursrv/serve"
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 )
 
 type CLI struct {
-	Serve     serve.CLI     `cmd:"" default:""`
-	Aggregate aggregate.CLI `cmd:""`
+	Serve serve.CLI `cmd:"" default:""`
 }
 
 func main() {
-	log.SetFlags(log.Ltime | log.Ldate | log.Lshortfile)
-	log.SetOutput(os.Stdout)
+	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
+		Level: slog.LevelInfo,
+	})))
 
 	var cli CLI
 	ctx := kong.Parse(&cli)

cmd/infra/ursrv/serve/analytics.go

@@ -1,276 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package serve
-
-import (
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-)
-
-type analytic struct {
-	Key        string
-	Count      int
-	Percentage float64
-	Items      []analytic `json:",omitempty"`
-}
-
-type analyticList []analytic
-
-func (l analyticList) Less(a, b int) bool {
-	if l[a].Key == "Others" {
-		return false
-	}
-	if l[b].Key == "Others" {
-		return true
-	}
-	return l[b].Count < l[a].Count // inverse
-}
-
-func (l analyticList) Swap(a, b int) {
-	l[a], l[b] = l[b], l[a]
-}
-
-func (l analyticList) Len() int {
-	return len(l)
-}
-
-// Returns a list of frequency analytics for a given list of strings.
-func analyticsFor(ss []string, cutoff int) []analytic {
-	m := make(map[string]int)
-	t := 0
-	for _, s := range ss {
-		m[s]++
-		t++
-	}
-
-	l := make([]analytic, 0, len(m))
-	for k, c := range m {
-		l = append(l, analytic{
-			Key:        k,
-			Count:      c,
-			Percentage: 100 * float64(c) / float64(t),
-		})
-	}
-
-	sort.Sort(analyticList(l))
-
-	if cutoff > 0 && len(l) > cutoff {
-		c := 0
-		for _, i := range l[cutoff:] {
-			c += i.Count
-		}
-		l = append(l[:cutoff], analytic{
-			Key:        "Others",
-			Count:      c,
-			Percentage: 100 * float64(c) / float64(t),
-		})
-	}
-
-	return l
-}
-
-// Find the points at which certain penetration levels are met
-func penetrationLevels(as []analytic, points []float64) []analytic {
-	sort.Slice(as, func(a, b int) bool {
-		return versionLess(as[b].Key, as[a].Key)
-	})
-
-	var res []analytic
-
-	idx := 0
-	sum := 0.0
-	for _, a := range as {
-		sum += a.Percentage
-		if sum >= points[idx] {
-			a.Count = int(points[idx])
-			a.Percentage = sum
-			res = append(res, a)
-			idx++
-			if idx == len(points) {
-				break
-			}
-		}
-	}
-	return res
-}
-
-func statsForInts(data []int) [4]float64 {
-	var res [4]float64
-	if len(data) == 0 {
-		return res
-	}
-
-	sort.Ints(data)
-	res[0] = float64(data[int(float64(len(data))*0.05)])
-	res[1] = float64(data[len(data)/2])
-	res[2] = float64(data[int(float64(len(data))*0.95)])
-	res[3] = float64(data[len(data)-1])
-	return res
-}
-
-func statsForInt64s(data []int64) [4]float64 {
-	var res [4]float64
-	if len(data) == 0 {
-		return res
-	}
-
-	sort.Slice(data, func(a, b int) bool {
-		return data[a] < data[b]
-	})
-
-	res[0] = float64(data[int(float64(len(data))*0.05)])
-	res[1] = float64(data[len(data)/2])
-	res[2] = float64(data[int(float64(len(data))*0.95)])
-	res[3] = float64(data[len(data)-1])
-	return res
-}
-
-func statsForFloats(data []float64) [4]float64 {
-	var res [4]float64
-	if len(data) == 0 {
-		return res
-	}
-
-	sort.Float64s(data)
-	res[0] = data[int(float64(len(data))*0.05)]
-	res[1] = data[len(data)/2]
-	res[2] = data[int(float64(len(data))*0.95)]
-	res[3] = data[len(data)-1]
-	return res
-}
-
-func group(by func(string) string, as []analytic, perGroup int, otherPct float64) []analytic {
-	var res []analytic
-
-next:
-	for _, a := range as {
-		group := by(a.Key)
-		for i := range res {
-			if res[i].Key == group {
-				res[i].Count += a.Count
-				res[i].Percentage += a.Percentage
-				if len(res[i].Items) < perGroup {
-					res[i].Items = append(res[i].Items, a)
-				}
-				continue next
-			}
-		}
-		res = append(res, analytic{
-			Key:        group,
-			Count:      a.Count,
-			Percentage: a.Percentage,
-			Items:      []analytic{a},
-		})
-	}
-
-	sort.Sort(analyticList(res))
-
-	if otherPct > 0 {
-		// Groups with less than otherPCt go into "Other"
-		other := analytic{
-			Key: "Other",
-		}
-		for i := 0; i < len(res); i++ {
-			if res[i].Percentage < otherPct || res[i].Key == "Other" {
-				other.Count += res[i].Count
-				other.Percentage += res[i].Percentage
-				res = append(res[:i], res[i+1:]...)
-				i--
-			}
-		}
-		if other.Count > 0 {
-			res = append(res, other)
-		}
-	}
-
-	return res
-}
-
-func byVersion(s string) string {
-	parts := strings.Split(s, ".")
-	if len(parts) >= 2 {
-		return strings.Join(parts[:2], ".")
-	}
-	return s
-}
-
-func byPlatform(s string) string {
-	parts := strings.Split(s, "-")
-	if len(parts) >= 2 {
-		return parts[0]
-	}
-	return s
-}
-
-var numericGoVersion = regexp.MustCompile(`^go[0-9]\.[0-9]+`)
-
-func byCompiler(s string) string {
-	if m := numericGoVersion.FindString(s); m != "" {
-		return m
-	}
-	return "Other"
-}
-
-func versionLess(a, b string) bool {
-	arel, apre := versionParts(a)
-	brel, bpre := versionParts(b)
-
-	minlen := len(arel)
-	if l := len(brel); l < minlen {
-		minlen = l
-	}
-
-	for i := 0; i < minlen; i++ {
-		if arel[i] != brel[i] {
-			return arel[i] < brel[i]
-		}
-	}
-
-	// Longer version is newer, when the preceding parts are equal
-	if len(arel) != len(brel) {
-		return len(arel) < len(brel)
-	}
-
-	if apre != bpre {
-		// "(+dev)" versions are ahead
-		if apre == plusStr {
-			return false
-		}
-		if bpre == plusStr {
-			return true
-		}
-		return apre < bpre
-	}
-
-	// don't actually care how the prerelease stuff compares for our purposes
-	return false
-}
-
-// Split a version as returned from transformVersion into parts.
-// "1.2.3-beta.2" -> []int{1, 2, 3}, "beta.2"}
-func versionParts(v string) ([]int, string) {
-	parts := strings.SplitN(v[1:], " ", 2) // " (+dev)" versions
-	if len(parts) == 1 {
-		parts = strings.SplitN(parts[0], "-", 2) // "-rc.1" type versions
-	}
-	fields := strings.Split(parts[0], ".")
-
-	release := make([]int, len(fields))
-	for i, s := range fields {
-		v, _ := strconv.Atoi(s)
-		release[i] = v
-	}
-
-	var prerelease string
-	if len(parts) > 1 {
-		prerelease = parts[1]
-	}
-
-	return release, prerelease
-}

cmd/infra/ursrv/serve/formatting.go

@@ -1,131 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package serve
-
-import (
-	"bytes"
-	"fmt"
-	"strings"
-)
-
-type NumberType int
-
-const (
-	NumberMetric NumberType = iota
-	NumberBinary
-	NumberDuration
-)
-
-func number(ntype NumberType, v float64) string {
-	switch ntype {
-	case NumberMetric:
-		return metric(v)
-	case NumberDuration:
-		return duration(v)
-	case NumberBinary:
-		return binary(v)
-	default:
-		return metric(v)
-	}
-}
-
-type suffix struct {
-	Suffix     string
-	Multiplier float64
-}
-
-var metricSuffixes = []suffix{
-	{"G", 1e9},
-	{"M", 1e6},
-	{"k", 1e3},
-}
-
-var binarySuffixes = []suffix{
-	{"Gi", 1 << 30},
-	{"Mi", 1 << 20},
-	{"Ki", 1 << 10},
-}
-
-var durationSuffix = []suffix{
-	{"year", 365 * 24 * 60 * 60},
-	{"month", 30 * 24 * 60 * 60},
-	{"day", 24 * 60 * 60},
-	{"hour", 60 * 60},
-	{"minute", 60},
-	{"second", 1},
-}
-
-func metric(v float64) string {
-	return withSuffix(v, metricSuffixes, false)
-}
-
-func binary(v float64) string {
-	return withSuffix(v, binarySuffixes, false)
-}
-
-func duration(v float64) string {
-	return withSuffix(v, durationSuffix, true)
-}
-
-func withSuffix(v float64, ps []suffix, pluralize bool) string {
-	for _, p := range ps {
-		if v >= p.Multiplier {
-			suffix := p.Suffix
-			if pluralize && v/p.Multiplier != 1.0 {
-				suffix += "s"
-			}
-			// If the number only has decimal zeroes, strip em off.
-			num := strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.1f", v/p.Multiplier), "0"), ".")
-			return fmt.Sprintf("%s %s", num, suffix)
-		}
-	}
-	return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.1f", v), "0"), ".")
-}
-
-// commatize returns a number with sep as thousands separators. Handles
-// integers and plain floats.
-func commatize(sep, s string) string {
-	// If no dot, don't do anything.
-	if !strings.ContainsRune(s, '.') {
-		return s
-	}
-	var b bytes.Buffer
-	fs := strings.SplitN(s, ".", 2)
-
-	l := len(fs[0])
-	for i := range fs[0] {
-		b.Write([]byte{s[i]})
-		if i < l-1 && (l-i)%3 == 1 {
-			b.WriteString(sep)
-		}
-	}
-
-	if len(fs) > 1 && len(fs[1]) > 0 {
-		b.WriteString(".")
-		b.WriteString(fs[1])
-	}
-
-	return b.String()
-}
-
-func proportion(m map[string]int, count int) float64 {
-	total := 0
-	isMax := true
-	for _, n := range m {
-		total += n
-		if n > count {
-			isMax = false
-		}
-	}
-	pct := (100 * float64(count)) / float64(total)
-	// To avoid rounding errors in the template, surpassing 100% and breaking
-	// the progress bars.
-	if isMax && len(m) > 1 && count != total {
-		pct -= 0.01
-	}
-	return pct
-}

cmd/infra/ursrv/serve/metrics.go

@@ -11,16 +11,51 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
-var metricReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
-	Namespace: "syncthing",
-	Subsystem: "ursrv",
-	Name:      "reports_total",
-}, []string{"version"})
+var (
+	metricReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "ursrv_v2",
+		Name:      "incoming_reports_total",
+	}, []string{"result"})
+	metricsCollectsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "ursrv_v2",
+		Name:      "collects_total",
+	})
+	metricsCollectSecondsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "ursrv_v2",
+		Name:      "collect_seconds_total",
+	})
+	metricsCollectSecondsLast = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: "syncthing",
+		Subsystem: "ursrv_v2",
+		Name:      "collect_seconds_last",
+	})
+	metricsRecalcsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "ursrv_v2",
+		Name:      "recalcs_total",
+	})
+	metricsRecalcSecondsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: "syncthing",
+		Subsystem: "ursrv_v2",
+		Name:      "recalc_seconds_total",
+	})
+	metricsRecalcSecondsLast = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: "syncthing",
+		Subsystem: "ursrv_v2",
+		Name:      "recalc_seconds_last",
+	})
+	metricsWriteSecondsLast = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: "syncthing",
+		Subsystem: "ursrv_v2",
+		Name:      "write_seconds_last",
+	})
+)
 
 func init() {
 	metricReportsTotal.WithLabelValues("fail")
-	metricReportsTotal.WithLabelValues("duplicate")
-	metricReportsTotal.WithLabelValues("v1")
-	metricReportsTotal.WithLabelValues("v2")
-	metricReportsTotal.WithLabelValues("v3")
+	metricReportsTotal.WithLabelValues("replace")
+	metricReportsTotal.WithLabelValues("accept")
 }

cmd/infra/ursrv/serve/prometheus.go

@@ -0,0 +1,351 @@
+// Copyright (C) 2024 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package serve
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"reflect"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/syncthing/syncthing/lib/ur/contract"
+)
+
+const namePrefix = "syncthing_usage_"
+
+type metricsSet struct {
+	srv *server
+
+	gauges         map[string]prometheus.Gauge
+	gaugeVecs      map[string]*prometheus.GaugeVec
+	gaugeVecLabels map[string][]string
+	summaries      map[string]*metricSummary
+
+	collectMut    sync.RWMutex
+	collectCutoff time.Duration
+}
+
+func newMetricsSet(srv *server) *metricsSet {
+	s := &metricsSet{
+		srv:            srv,
+		gauges:         make(map[string]prometheus.Gauge),
+		gaugeVecs:      make(map[string]*prometheus.GaugeVec),
+		gaugeVecLabels: make(map[string][]string),
+		summaries:      make(map[string]*metricSummary),
+		collectCutoff:  -24 * time.Hour,
+	}
+
+	var initForType func(reflect.Type)
+	initForType = func(t reflect.Type) {
+		for i := range t.NumField() {
+			field := t.Field(i)
+			if field.Type.Kind() == reflect.Struct {
+				initForType(field.Type)
+				continue
+			}
+			name, typ, label := fieldNameTypeLabel(field)
+			sname, labels := nameConstLabels(name)
+			switch typ {
+			case "gauge":
+				s.gauges[name] = prometheus.NewGauge(prometheus.GaugeOpts{
+					Name:        namePrefix + sname,
+					ConstLabels: labels,
+				})
+			case "summary":
+				s.summaries[name] = newMetricSummary(namePrefix+sname, nil, labels)
+			case "gaugeVec":
+				s.gaugeVecLabels[name] = append(s.gaugeVecLabels[name], label)
+			case "summaryVec":
+				s.summaries[name] = newMetricSummary(namePrefix+sname, []string{label}, labels)
+			}
+		}
+	}
+	initForType(reflect.ValueOf(contract.Report{}).Type())
+
+	for name, labels := range s.gaugeVecLabels {
+		s.gaugeVecs[name] = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: namePrefix + name,
+		}, labels)
+	}
+
+	return s
+}
+
+func fieldNameTypeLabel(rf reflect.StructField) (string, string, string) {
+	metric := rf.Tag.Get("metric")
+	name, typ, ok := strings.Cut(metric, ",")
+	if !ok {
+		return "", "", ""
+	}
+	gv, label, ok := strings.Cut(typ, ":")
+	if ok {
+		typ = gv
+	}
+	return name, typ, label
+}
+
+func nameConstLabels(name string) (string, prometheus.Labels) {
+	if name == "-" {
+		return "", nil
+	}
+	name, labels, ok := strings.Cut(name, "{")
+	if !ok {
+		return name, nil
+	}
+	lls := strings.Split(labels[:len(labels)-1], ",")
+	m := make(map[string]string)
+	for _, l := range lls {
+		k, v, _ := strings.Cut(l, "=")
+		m[k] = v
+	}
+	return name, m
+}
+
+func (s *metricsSet) Serve(ctx context.Context) error {
+	s.recalc()
+
+	const recalcInterval = 5 * time.Minute
+	next := time.Until(time.Now().Truncate(recalcInterval).Add(recalcInterval))
+	recalcTimer := time.NewTimer(next)
+	defer recalcTimer.Stop()
+
+	for {
+		select {
+		case <-recalcTimer.C:
+			s.recalc()
+			next := time.Until(time.Now().Truncate(recalcInterval).Add(recalcInterval))
+			recalcTimer.Reset(next)
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	}
+}
+
+func (s *metricsSet) recalc() {
+	s.collectMut.Lock()
+	defer s.collectMut.Unlock()
+
+	t0 := time.Now()
+	defer func() {
+		dur := time.Since(t0)
+		slog.Info("Metrics recalculated", "d", dur.String())
+		metricsRecalcSecondsLast.Set(dur.Seconds())
+		metricsRecalcSecondsTotal.Add(dur.Seconds())
+		metricsRecalcsTotal.Inc()
+	}()
+
+	for _, g := range s.gauges {
+		g.Set(0)
+	}
+	for _, g := range s.gaugeVecs {
+		g.Reset()
+	}
+	for _, g := range s.summaries {
+		g.Reset()
+	}
+
+	cutoff := time.Now().Add(s.collectCutoff)
+	s.srv.reports.Range(func(key string, r *contract.Report) bool {
+		if s.collectCutoff < 0 && r.Received.Before(cutoff) {
+			s.srv.reports.Delete(key)
+			return true
+		}
+		s.addReport(r)
+		return true
+	})
+}
+
+func (s *metricsSet) addReport(r *contract.Report) {
+	gaugeVecs := make(map[string][]string)
+	s.addReportStruct(reflect.ValueOf(r).Elem(), gaugeVecs)
+	for name, lv := range gaugeVecs {
+		s.gaugeVecs[name].WithLabelValues(lv...).Add(1)
+	}
+}
+
+func (s *metricsSet) addReportStruct(v reflect.Value, gaugeVecs map[string][]string) {
+	t := v.Type()
+	for i := range v.NumField() {
+		field := v.Field(i)
+		if field.Kind() == reflect.Struct {
+			s.addReportStruct(field, gaugeVecs)
+			continue
+		}
+
+		name, typ, label := fieldNameTypeLabel(t.Field(i))
+		switch typ {
+		case "gauge":
+			switch v := field.Interface().(type) {
+			case int:
+				s.gauges[name].Add(float64(v))
+			case string:
+				s.gaugeVecs[name].WithLabelValues(v).Add(1)
+			case bool:
+				if v {
+					s.gauges[name].Add(1)
+				}
+			}
+		case "gaugeVec":
+			var labelValue string
+			switch v := field.Interface().(type) {
+			case string:
+				labelValue = v
+			case int:
+				labelValue = strconv.Itoa(v)
+			case map[string]int:
+				for k, v := range v {
+					labelValue = k
+					field.SetInt(int64(v))
+					break
+				}
+			}
+			if _, ok := gaugeVecs[name]; !ok {
+				gaugeVecs[name] = make([]string, len(s.gaugeVecLabels[name]))
+			}
+			for i, l := range s.gaugeVecLabels[name] {
+				if l == label {
+					gaugeVecs[name][i] = labelValue
+					break
+				}
+			}
+		case "summary", "summaryVec":
+			switch v := field.Interface().(type) {
+			case int:
+				s.summaries[name].Observe("", float64(v))
+			case float64:
+				s.summaries[name].Observe("", v)
+			case []int:
+				for _, v := range v {
+					s.summaries[name].Observe("", float64(v))
+				}
+			case map[string]int:
+				for k, v := range v {
+					if k == "" {
+						// avoid empty string labels as those are the sign
+						// of a non-vec summary
+						k = "unknown"
+					}
+					s.summaries[name].Observe(k, float64(v))
+				}
+			}
+		}
+	}
+}
+
+func (s *metricsSet) Describe(c chan<- *prometheus.Desc) {
+	for _, g := range s.gauges {
+		g.Describe(c)
+	}
+	for _, g := range s.gaugeVecs {
+		g.Describe(c)
+	}
+	for _, g := range s.summaries {
+		g.Describe(c)
+	}
+}
+
+func (s *metricsSet) Collect(c chan<- prometheus.Metric) {
+	s.collectMut.RLock()
+	defer s.collectMut.RUnlock()
+
+	t0 := time.Now()
+	defer func() {
+		dur := time.Since(t0).Seconds()
+		metricsCollectSecondsLast.Set(dur)
+		metricsCollectSecondsTotal.Add(dur)
+		metricsCollectsTotal.Inc()
+	}()
+
+	for _, g := range s.gauges {
+		c <- g
+	}
+	for _, g := range s.gaugeVecs {
+		g.Collect(c)
+	}
+	for _, g := range s.summaries {
+		g.Collect(c)
+	}
+}
+
+type metricSummary struct {
+	name   string
+	values map[string][]float64
+	zeroes map[string]int
+
+	qDesc     *prometheus.Desc
+	countDesc *prometheus.Desc
+	sumDesc   *prometheus.Desc
+	zDesc     *prometheus.Desc
+}
+
+func newMetricSummary(name string, labels []string, constLabels prometheus.Labels) *metricSummary {
+	return &metricSummary{
+		name:      name,
+		values:    make(map[string][]float64),
+		zeroes:    make(map[string]int),
+		qDesc:     prometheus.NewDesc(name, "", append(labels, "quantile"), constLabels),
+		countDesc: prometheus.NewDesc(name+"_nonzero_count", "", labels, constLabels),
+		sumDesc:   prometheus.NewDesc(name+"_sum", "", labels, constLabels),
+		zDesc:     prometheus.NewDesc(name+"_zero_count", "", labels, constLabels),
+	}
+}
+
+func (q *metricSummary) Observe(labelValue string, v float64) {
+	if v == 0 {
+		q.zeroes[labelValue]++
+		return
+	}
+	q.values[labelValue] = append(q.values[labelValue], v)
+}
+
+func (q *metricSummary) Describe(c chan<- *prometheus.Desc) {
+	c <- q.qDesc
+	c <- q.countDesc
+	c <- q.sumDesc
+	c <- q.zDesc
+}
+
+func (q *metricSummary) Collect(c chan<- prometheus.Metric) {
+	for lv, vs := range q.values {
+		var labelVals []string
+		if lv != "" {
+			labelVals = []string{lv}
+		}
+
+		c <- prometheus.MustNewConstMetric(q.countDesc, prometheus.GaugeValue, float64(len(vs)), labelVals...)
+		c <- prometheus.MustNewConstMetric(q.zDesc, prometheus.GaugeValue, float64(q.zeroes[lv]), labelVals...)
+
+		var sum float64
+		for _, v := range vs {
+			sum += v
+		}
+		c <- prometheus.MustNewConstMetric(q.sumDesc, prometheus.GaugeValue, sum, labelVals...)
+
+		if len(vs) == 0 {
+			return
+		}
+
+		slices.Sort(vs)
+
+		pctiles := []float64{0, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.975, 0.99, 1}
+		for _, pct := range pctiles {
+			idx := int(float64(len(vs)-1) * pct)
+			c <- prometheus.MustNewConstMetric(q.qDesc, prometheus.GaugeValue, vs[idx], append(labelVals, fmt.Sprint(pct))...)
+		}
+	}
+}
+
+func (q *metricSummary) Reset() {
+	clear(q.values)
+	clear(q.zeroes)
+}

cmd/infra/ursrv/serve/static/index.html

@@ -1,623 +0,0 @@
-<!DOCTYPE html>
-<!--
-Copyright (C) 2014 Jakob Borg and other contributors. All rights reserved.
-Use of this source code is governed by an MIT-style license that can be
-found in the LICENSE file.
--->
-<html lang="en">
-<head>
-  <meta charset="utf-8">
-  <meta http-equiv="X-UA-Compatible" content="IE=edge">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <meta name="description" content="">
-  <meta name="author" content="">
-  <link rel="shortcut icon" href="static/assets/img/favicon.png">
-
-  <title>Syncthing Usage Reports</title>
-  <link href="static/bootstrap/css/bootstrap.min.css" rel="stylesheet">
-  <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
-  <script type="text/javascript" src="static/bootstrap/js/bootstrap.min.js"></script>
-  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/leaflet/0.7.7/leaflet.css">
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/leaflet/0.7.7/leaflet.js"></script>
-  <script src="https://cdn.jsdelivr.net/npm/heatmapjs@2.0.2/heatmap.min.js"></script>
-  <script src="https://cdn.jsdelivr.net/npm/leaflet-heatmap@1.0.0/leaflet-heatmap.js"></script>
-
-  <style type="text/css">
-    body {
-      margin: 40px;
-      font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
-    }
-    tr.main td {
-      font-weight: bold;
-    }
-    tr.child td.first {
-      padding-left: 2em;
-    }
-    .progress-bar {
-      overflow:hidden;
-      white-space:nowrap;
-      text-overflow: ellipsis;
-    }
-  </style>
-  <script type="text/javascript"
-          src='https://www.google.com/jsapi?autoload={
-            "modules":[{
-              "name":"visualization",
-              "version":"1",
-              "packages":["corechart"]
-            }]
-          }'></script>
-
-  <script type="text/javascript">
-    google.setOnLoadCallback(drawVersionChart);
-    google.setOnLoadCallback(drawBlockStatsChart);
-    google.setOnLoadCallback(drawPerformanceCharts);
-
-    function drawVersionChart() {
-      // Summary version chart for versions that at some point in the chart
-      // reaches 250 devices. This filters out versions that are old and
-      // uninteresting yet linger forever with like four users.
-      var jsonData = $.ajax({url: "summary.json?min=250", dataType:"json", async: false}).responseText;
-      var rows = JSON.parse(jsonData);
-
-      var data = new google.visualization.DataTable();
-      data.addColumn('date', 'Day');
-      for (var i = 1; i < rows[0].length; i++){
-        data.addColumn('number', rows[0][i]);
-      }
-      for (var i = 1; i < rows.length; i++){
-        rows[i][0] = new Date(rows[i][0]);
-        data.addRow(rows[i]);
-      };
-
-      var options = {
-        legend: { position: 'bottom', alignment: 'center' },
-        isStacked: true,
-        colors: ['rgb(102,194,165)','rgb(252,141,98)','rgb(141,160,203)','rgb(231,138,195)','rgb(166,216,84)','rgb(255,217,47)'],
-        chartArea: {left: 80, top: 20, width: '1020', height: '300'},
-      };
-
-      var chart = new google.visualization.AreaChart(document.getElementById('versionChart'));
-      chart.draw(data, options);
-    }
-
-    function formatGibibytes(gibibytes, decimals) {
-      if(gibibytes == 0) return '0 GiB';
-      var k = 1024,
-        dm = decimals || 2,
-        sizes = ['GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'],
-        i = Math.floor(Math.log(gibibytes) / Math.log(k));
-      if (i < 0) {
-        sizes = 'MiB';
-      } else {
-        sizes = sizes[i];
-      }
-      return parseFloat((gibibytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes;
-    }
-
-
-    function drawBlockStatsChart() {
-      var jsonData = $.ajax({url: "blockstats.json", dataType:"json", async: false}).responseText;
-      var rows = JSON.parse(jsonData);
-
-      var data = new google.visualization.DataTable();
-      data.addColumn('date', 'Day');
-      for (var i = 1; i < rows[0].length; i++){
-        data.addColumn('number', rows[0][i]);
-      }
-
-      var totals = [0, 0, 0, 0, 0, 0];
-      for (var i = 1; i < rows.length; i++){
-        rows[i][0] = new Date(rows[i][0]);
-        for (var j = 2; j < rows[i].length; j++) {
-          totals[j-2] += rows[i][j];
-        }
-        data.addRow(rows[i]);
-      };
-
-      var totalTotals = totals.reduce(function(a, b) { return a + b; }, 0);
-
-      if (totalTotals > 0) {
-        var content = "<table class='table'>\n"
-        for (var j = 2; j < rows[0].length; j++) {
-          content += "<tr><td><b>" + rows[0][j].replace(' (GiB)', '') + "</b></td><td>" + formatGibibytes(totals[j-2].toFixed(2)) + " (" + ((100*totals[j-2])/totalTotals).toFixed(2) +"%)</td></tr>\n";
-        }
-        content += "</table>";
-        document.getElementById("data-to-date").innerHTML = content;
-      } else {
-        // No data, hide it.
-        document.getElementById("block-stats").outerHTML  = "";
-        return;
-      }
-
-      var options = {
-        focusTarget: 'category',
-        vAxes: {0: {}, 1: {}},
-        series: {0: {type: 'line', targetAxisIndex:1}},
-        isStacked: true,
-        legend: {position: 'none'},
-        colors: ['rgb(102,194,165)','rgb(252,141,98)','rgb(141,160,203)','rgb(231,138,195)','rgb(166,216,84)','rgb(255,217,47)'],
-        chartArea: {left: 80, top: 20, width: '1020', height: '300'},
-      };
-
-      var chart = new google.visualization.AreaChart(document.getElementById('blockStatsChart'));
-      chart.draw(data, options);
-    }
-
-    function drawPerformanceCharts() {
-      var jsonData = $.ajax({url: "/performance.json", dataType:"json", async: false}).responseText;
-      var rows = JSON.parse(jsonData);
-      for (var i = 1; i < rows.length; i++){
-        rows[i][0] = new Date(rows[i][0]);
-      }
-
-      drawChart(rows, 1, 'Total Number of Files', 'totFilesChart', 1e6, 1);
-      drawChart(rows, 2, 'Total Folder Size (GiB)', 'totMiBChart', 1e6, 1024);
-      drawChart(rows, 3, 'Hash Performance (MiB/s)', 'hashPerfChart', 1000, 1);
-      drawChart(rows, 4, 'System RAM Size (GiB)', 'memSizeChart', 1e6, 1024);
-      drawChart(rows, 5, 'Memory Usage (MiB)', 'memUsageChart', 250, 1);
-    }
-
-    function drawChart(rows, index, title, id, cutoff, divisor) {
-      var data = new google.visualization.DataTable();
-      data.addColumn('date', 'Day');
-      data.addColumn('number', title);
-
-      var row;
-      for (var i = 1; i < rows.length; i++){
-          row = [rows[i][0], rows[i][index] / divisor];
-        if (row[1] > cutoff) {
-          row[1] = null;
-        }
-        data.addRow(row);
-      }
-
-      var options = {
-        legend: { position: 'bottom', alignment: 'center' },
-        colors: ['rgb(102,194,165)','rgb(252,141,98)','rgb(141,160,203)','rgb(231,138,195)','rgb(166,216,84)','rgb(255,217,47)'],
-        chartArea: {left: 80, top: 20, width: '1020', height: '300'},
-        vAxes: {0: {minValue: 0}},
-      };
-
-      var chart = new google.visualization.LineChart(document.getElementById(id));
-      chart.draw(data, options);
-    }
-
-    var locations = [];
-    {{range $location, $weight := .locations}}
-    locations.push({lat:{{- $location.Latitude -}},lng:{{- $location.Longitude -}},count:Math.min(100, {{- $weight -}})});
-    {{- end}}
-
-    function drawHeatMap() {
-      if (locations.length == 0) {
-        return;
-      }
-      var testData = {
-        data: locations
-      };
-
-      var baseLayer = L.tileLayer(
-        'https://tile.openstreetmap.org/{z}/{x}/{y}.png',{
-          attribution: '...',
-          maxZoom: 18
-        }
-      );
-      var cfg = {
-        "radius": 1,
-        "minOpacity": .25,
-        "maxOpacity": .8,
-        "scaleRadius": true,
-        "useLocalExtrema": true,
-        latField: 'lat',
-        lngField: 'lng',
-        valueField: 'count',
-        gradient: {
-            '.1': 'cyan',
-            '.8': 'blue',
-            '.95': 'red'
-          }
-      };
-      var heatmapLayer = new HeatmapOverlay(cfg);
-
-      var map = new L.Map('map', {
-        center: new L.LatLng(25, 0),
-        zoom: 1,
-        layers: [baseLayer, heatmapLayer]
-      });
-      heatmapLayer.setData(testData);
-    }
-  </script>
-</head>
-
-<body>
-  <div class="container">
-    <div class="row">
-      <div class="col-md-12">
-        <h1>Syncthing Usage Data</h1>
-
-        <h4 id="active-users">Active Users per Day and Version</h4>
-        <p>
-          This is the total number of unique users with reporting enabled, per day. Area color represents the major version.
-        </p>
-        <div class="img-thumbnail" id="versionChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
-
-        <div id="block-stats">
-          <h4>Data Transfers per Day</h4>
-          <p>
-            This is total data transferred per day. Also shows how much data was saved (not transferred) by each of the methods syncthing uses.
-          </p>
-          <div class="img-thumbnail" id="blockStatsChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
-          <h4 id="totals-to-date">Totals to date</h4>
-          <p id="data-to-date">
-            No data
-          </p>
-        </div>
-
-        <h4 id="metrics">Usage Metrics</h4>
-        <p>
-          This is the aggregated usage report data for the last 24 hours. Data based on <b>{{.nodes}}</b> devices that have reported in.
-        </p>
-
-        {{if .locations}}
-        <div class="img-thumbnail" id="map" style="width: 1130px; height: 400px; padding: 10px;"></div>
-        <p class="text-muted">
-         Heatmap max intensity is capped at 100 reports within a location.
-        </p>
-        <div class="panel panel-default">
-          <div class="panel-heading">
-            <h4 class="panel-title">
-              <a data-toggle="collapse" href="#collapseTwo">Break down per country</a>
-            </h4>
-          </div>
-          <div id="collapseTwo" class="panel-collapse collapse">
-            <div class="panel-body less-padding">
-              <div class="row">
-                <div class="col-md-6">
-                  <table class="table table-striped">
-                    <tbody>
-                      {{range .countries | slice 2 1}}
-                      <tr>
-                        <td style="width: 45%">{{.Key}}</td>
-                        <td style="width: 5%" class="text-right">{{if ge .Pct 10.0}}{{.Pct | printf "%.0f"}}{{else if ge .Pct 1.0}}{{.Pct | printf "%.01f"}}{{else}}{{.Pct | printf "%.02f"}}{{end}}%</td>
-                        <td style="width: 5%" class="text-right">{{.Count}}</td>
-                        <td>
-                            <div class="progress-bar" role="progressbar" aria-valuenow="{{.Pct | printf "%.02f"}}" aria-valuemin="0" aria-valuemax="100" style="width: {{.Pct | printf "%.02f"}}%; height:20px"></div>
-                        </td>
-                      </tr>
-                      {{end}}
-                    </tbody>
-                  </table>
-                </div>
-                <div class="col-md-6">
-                  <table class="table table-striped">
-                    <tbody>
-                      {{range .countries | slice 2 2}}
-                      <tr>
-                        <td style="width: 45%">{{.Key}}</td>
-                        <td style="width: 5%" class="text-right">{{if ge .Pct 10.0}}{{.Pct | printf "%.0f"}}{{else if ge .Pct 1.0}}{{.Pct | printf "%.01f"}}{{else}}{{.Pct | printf "%.02f"}}{{end}}%</td>
-                        <td style="width: 5%" class="text-right">{{.Count}}</td>
-                        <td>
-                            <div class="progress-bar" role="progressbar" aria-valuenow="{{.Pct | printf "%.02f"}}" aria-valuemin="0" aria-valuemax="100" style="width: {{.Pct | printf "%.02f"}}%; height:20px"></div>
-                        </td>
-                      </tr>
-                      {{end}}
-                    </tbody>
-                  </table>
-                </div>
-              </div>
-            </div>
-          </div>
-        </div>
-        {{end}}
-        <table class="table table-striped">
-          <thead>
-            <tr>
-              <th></th>
-              <th colspan="4" class="text-center">
-                <a href="https://en.wikipedia.org/wiki/Percentile">Percentile</a>
-              </th>
-            </tr>
-            <tr>
-              <th></th>
-              <th class="text-right">5%</th>
-              <th class="text-right">50%</th>
-              <th class="text-right">95%</th>
-              <th class="text-right">100%</th>
-            </tr>
-          </thead>
-          <tbody>
-            {{range .categories}}
-            <tr>
-              <td>{{.Descr}}</td>
-              <td class="text-right">{{index .Values 0 | number .Type | commatize " "}}{{.Unit}}</td>
-              <td class="text-right">{{index .Values 1 | number .Type | commatize " "}}{{.Unit}}</td>
-              <td class="text-right">{{index .Values 2 | number .Type | commatize " "}}{{.Unit}}</td>
-              <td class="text-right">{{index .Values 3 | number .Type | commatize " "}}{{.Unit}}</td>
-            </tr>
-            {{end}}
-          </tbody>
-        </table>
-      </div>
-    </div>
-
-    <div class="row">
-      <div class="col-md-6">
-        <table class="table table-striped">
-          <thead>
-            <tr>
-              <th>Version</th><th class="text-right">Devices</th><th class="text-right">Share</th>
-            </tr>
-          </thead>
-          <tbody>
-            {{range .versions}}
-              {{if gt .Percentage 0.1}}
-                <tr class="main">
-                  <td>{{.Key}}</td>
-                  <td class="text-right">{{.Count}}</td>
-                  <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
-                </tr>
-                {{range .Items}}
-                  {{if gt .Percentage 0.1}}
-                    <tr class="child">
-                      <td class="first">{{.Key}}</td>
-                      <td class="text-right">{{.Count}}</td>
-                      <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
-                    </tr>
-                  {{end}}
-                {{end}}
-              {{end}}
-            {{end}}
-          </tbody>
-        </table>
-        <table class="table table-striped">
-          <thead>
-            <tr>
-                <th>Penetration Level</th>
-                <th>Version</th>
-              <th class="text-right">Actual</th>
-            </tr>
-          </thead>
-          <tbody>
-            {{range .versionPenetrations}}
-            <tr>
-                <td>{{.Count}}%</td>
-                <td>&ge; {{.Key}}</td>
-              <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
-            </tr>
-            {{end}}
-          </tbody>
-        </table>
-      </div>
-
-      <div class="col-md-6">
-        <table class="table table-striped">
-          <thead>
-            <tr>
-              <th>Platform</th>
-              <th class="text-right">Devices</th>
-              <th class="text-right">Share</th>
-            </tr>
-          </thead>
-          <tbody>
-            {{range .platforms}}
-              <tr class="main">
-                <td>{{.Key}}</td>
-                <td class="text-right">{{.Count}}</td>
-                <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
-              </tr>
-              {{range .Items}}
-                <tr class="child">
-                  <td class="first">{{.Key}}</td>
-                  <td class="text-right">{{.Count}}</td>
-                  <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
-                </tr>
-              {{end}}
-            {{end}}
-          </tbody>
-        </table>
-      </div>
-
-    </div>
-    <div class="row">
-
-      <div class="col-md-6">
-        <table class="table table-striped">
-          <thead>
-            <tr>
-              <th>Compiler</th>
-              <th class="text-right">Devices</th>
-              <th class="text-right">Share</th>
-            </tr>
-          </thead>
-          <tbody>
-            {{range .compilers}}
-              <tr class="main">
-                <td>{{.Key}}</td>
-                <td class="text-right">{{.Count}}</td>
-                <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
-              </tr>
-              {{range .Items}}
-                {{if or (gt .Percentage 0.1) (eq .Key "Others")}}
-                  <tr class="child">
-                    <td class="first">{{.Key}}</td>
-                    <td class="text-right">{{.Count}}</td>
-                    <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
-                  </tr>
-                {{end}}
-              {{end}}
-            {{end}}
-          </tbody>
-        </table>
-      </div>
-
-      <div class="col-md-6">
-        <table class="table table-striped">
-          <thead>
-            <tr>
-              <th>Distribution Channel</th>
-              <th class="text-right">Devices</th>
-              <th class="text-right">Share</th>
-            </tr>
-          </thead>
-          <tbody>
-            {{range .distributions}}
-            <tr>
-              <td>{{.Key}}</td>
-              <td class="text-right">{{.Count}}</td>
-              <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
-            </tr>
-            {{end}}
-          </tbody>
-        </table>
-      </div>
-
-      <div class="col-md-6">
-        <table class="table table-striped">
-          <thead>
-            <tr>
-              <th>Builder</th>
-              <th class="text-right">Devices</th>
-              <th class="text-right">Share</th>
-            </tr>
-          </thead>
-          <tbody>
-            {{range .builders}}
-            <tr>
-              <td>{{.Key}}</td>
-              <td class="text-right">{{.Count}}</td>
-              <td class="text-right">{{.Percentage | printf "%.01f"}}%</td>
-            </tr>
-            {{end}}
-          </tbody>
-        </table>
-      </div>
-
-    </div>
-
-    <div class="row">
-        <div class="col-md-12">
-            <h4 id="features">Feature Usage</h4>
-            <p>
-                The following lists feature usage. Some features are reported per report, some are per sum of units within report (eg. devices with static addresses among all known devices per report).
-                Currently there are <b>{{.versionNodes.v2}}</b> devices reporting for version 2 and <b>{{.versionNodes.v3}}</b> for version 3.
-            </p>
-        </div>
-    </div>
-
-
-    <div class="row">
-    {{$i := counter}}
-    {{range $featureName := .featureOrder}}
-      {{$featureValues := index $.features $featureName }}
-      {{if $i.DrawTwoDivider}}
-        </div>
-        <div class="row">
-      {{end}}
-      {{ $i.Increment }}
-      <div class="col-md-6">
-          <table class="table table-striped">
-              <thead><tr>
-                  <th>{{$featureName}} Features</th><th colspan="2" class="text-center">Usage</th>
-              </tr></thead>
-              <tbody>
-                  {{range $featureValues}}
-                  <tr>
-                      <td style="width: 50%">{{.Key}} ({{.Version}})</td>
-                      <td style="width: 10%" class="text-right">{{if ge .Pct 10.0}}{{.Pct | printf "%.0f"}}{{else if ge .Pct 1.0}}{{.Pct | printf "%.01f"}}{{else}}{{.Pct | printf "%.02f"}}{{end}}%</td>
-                      <td style="width: 40%" {{if lt .Pct 5.0}}data-toggle="tooltip" title='{{.Count}}'{{end}}>
-                          <div class="progress-bar" role="progressbar" aria-valuenow="{{.Pct | printf "%.02f"}}" aria-valuemin="0" aria-valuemax="100" style="width: {{.Pct | printf "%.02f"}}%; height:20px" {{if ge .Pct 5.0}}data-toggle="tooltip" title='{{.Count}}'{{end}}></div>
-                      </td>
-                  </tr>
-                  {{end}}
-              </tbody>
-          </table>
-      </div>
-    {{end}}
-    </div>
-
-    <div class="row">
-        <div class="col-md-12">
-            <h4 id="features">Feature Group Usage</h4>
-            <p>
-                The following lists feature usage groups, which might include multiple occourances of a feature use per report.
-            </p>
-        </div>
-    </div>
-
-    <div class="row">
-    {{$i := counter}}
-    {{range $featureName := .featureOrder}}
-      {{$featureValues := index $.featureGroups $featureName }}
-      {{if $i.DrawTwoDivider}}
-        </div>
-        <div class="row">
-      {{end}}
-      {{ $i.Increment }}
-      <div class="col-md-6">
-          <table class="table table-striped">
-              <thead><tr>
-                  <th>{{$featureName}} Group Features</th><th colspan="2" class="text-center">Usage</th>
-              </tr></thead>
-              <tbody>
-                  {{range $featureValues}}
-                  {{$counts := .Counts}}
-                  <tr>
-                      <td style="width: 50%">
-                          <div data-toggle="tooltip" title='{{range $key, $value := .Counts}}{{$key}} ({{$value | proportion $counts | printf "%.02f"}}% - {{$value}})</br>{{end}}'>
-                              {{.Key}} ({{.Version}})
-                          </div>
-                      </td>
-                      <td style="width: 50%">
-                          <div class="progress" role="progressbar" style="width: 100%">
-                          {{$j := counter}}
-                          {{range $key, $value := .Counts}}
-                              {{with $valuePct := $value | proportion $counts}}
-                              <div class="progress-bar {{ $j.Current | progressBarClassByIndex }}" style='width: {{$valuePct | printf "%.02f"}}%' data-toggle="tooltip" title='{{$key}} ({{$valuePct | printf "%.02f"}}% - {{$value}})'>
-                                  {{if ge $valuePct 30.0}}{{$key}}{{end}}
-                              </div>
-                              {{end}}
-                              {{ $j.Increment }}
-                          {{end}}
-                          </div>
-                      </td>
-                  </tr>
-                  {{end}}
-              </tbody>
-          </table>
-      </div>
-    {{end}}
-    </div>
-     <div class="row">
-      <div class="col-md-12">
-        <h1 id="performance-charts">Historical Performance Data</h1>
-        <p>These charts are all the average of the corresponding metric, for the entire population of a given day.</p>
-
-        <h4 id="hash-performance">Hash Performance (MiB/s)</h4>
-        <div class="img-thumbnail" id="hashPerfChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
-
-        <h4 id="memory-usage">Memory Usage (MiB)</h4>
-        <div class="img-thumbnail" id="memUsageChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
-
-        <h4 id="total-files">Total Number of Files</h4>
-        <div class="img-thumbnail" id="totFilesChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
-
-        <h4 id="total-size">Total Folder Size (GiB)</h4>
-        <div class="img-thumbnail" id="totMiBChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
-
-        <h4 id="system-ram">System RAM Size (GiB)</h4>
-        <div class="img-thumbnail" id="memSizeChart" style="width: 1130px; height: 400px; padding: 10px;"></div>
-      </div>
-    </div>
-  </div>
-  <hr>
-  <p>
-    <a href="https://github.com/syncthing/syncthing/">Source code</a>.
-    This product includes GeoLite2 data created by MaxMind, available from
-    <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
-  </p>
-  <script type="text/javascript">
-    $('[data-toggle="tooltip"]').tooltip({html:true});
-    drawHeatMap();
-  </script>
-</body>
-</html>

cmd/stdiscosrv/amqp.go

@@ -10,19 +10,24 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"log"
+	"log/slog"
 
 	amqp "github.com/rabbitmq/amqp091-go"
-	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/thejerf/suture/v4"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/syncthing/syncthing/internal/gen/discosrv"
+	"github.com/syncthing/syncthing/internal/protoutil"
+	"github.com/syncthing/syncthing/lib/protocol"
 )
 
 type amqpReplicator struct {
 	suture.Service
+
 	broker   string
 	sender   *amqpSender
 	receiver *amqpReceiver
-	outbox   chan ReplicationRecord
+	outbox   chan *discosrv.ReplicationRecord
 }
 
 func newAMQPReplicator(broker, clientID string, db database) *amqpReplicator {
@@ -31,7 +36,7 @@ func newAMQPReplicator(broker, clientID string, db database) *amqpReplicator {
 	sender := &amqpSender{
 		broker:   broker,
 		clientID: clientID,
-		outbox:   make(chan ReplicationRecord, replicationOutboxSize),
+		outbox:   make(chan *discosrv.ReplicationRecord, replicationOutboxSize),
 	}
 	svc.Add(sender)
 
@@ -47,18 +52,18 @@ func newAMQPReplicator(broker, clientID string, db database) *amqpReplicator {
 		broker:   broker,
 		sender:   sender,
 		receiver: receiver,
-		outbox:   make(chan ReplicationRecord, replicationOutboxSize),
+		outbox:   make(chan *discosrv.ReplicationRecord, replicationOutboxSize),
 	}
 }
 
-func (s *amqpReplicator) send(key *protocol.DeviceID, ps []DatabaseAddress, seen int64) {
+func (s *amqpReplicator) send(key *protocol.DeviceID, ps []*discosrv.DatabaseAddress, seen int64) {
 	s.sender.send(key, ps, seen)
 }
 
 type amqpSender struct {
 	broker   string
 	clientID string
-	outbox   chan ReplicationRecord
+	outbox   chan *discosrv.ReplicationRecord
 }
 
 func (s *amqpSender) Serve(ctx context.Context) error {
@@ -73,12 +78,12 @@ func (s *amqpSender) Serve(ctx context.Context) error {
 	for {
 		select {
 		case rec := <-s.outbox:
-			size := rec.Size()
+			size := proto.Size(rec)
 			if len(buf) < size {
 				buf = make([]byte, size)
 			}
 
-			n, err := rec.MarshalTo(buf)
+			n, err := protoutil.MarshalTo(buf, rec)
 			if err != nil {
 				replicationSendsTotal.WithLabelValues("error").Inc()
 				return fmt.Errorf("replication marshal: %w", err)
@@ -111,8 +116,8 @@ func (s *amqpSender) String() string {
 	return fmt.Sprintf("amqpSender(%q)", s.broker)
 }
 
-func (s *amqpSender) send(key *protocol.DeviceID, ps []DatabaseAddress, seen int64) {
-	item := ReplicationRecord{
+func (s *amqpSender) send(key *protocol.DeviceID, ps []*discosrv.DatabaseAddress, seen int64) {
+	item := &discosrv.ReplicationRecord{
 		Key:       key[:],
 		Addresses: ps,
 		Seen:      seen,
@@ -158,8 +163,8 @@ func (s *amqpReceiver) Serve(ctx context.Context) error {
 				continue
 			}
 
-			var rec ReplicationRecord
-			if err := rec.Unmarshal(msg.Body); err != nil {
+			var rec discosrv.ReplicationRecord
+			if err := proto.Unmarshal(msg.Body, &rec); err != nil {
 				replicationRecvsTotal.WithLabelValues("error").Inc()
 				return fmt.Errorf("replication unmarshal: %w", err)
 			}
@@ -168,7 +173,7 @@ func (s *amqpReceiver) Serve(ctx context.Context) error {
 				id, err = protocol.DeviceIDFromString(string(rec.Key))
 			}
 			if err != nil {
-				log.Println("Replication device ID:", err)
+				slog.Warn("Failed to parse replication device ID", "error", err)
 				replicationRecvsTotal.WithLabelValues("error").Inc()
 				continue
 			}

cmd/stdiscosrv/apisrv.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	io "io"
 	"log"
+	"log/slog"
 	"math/rand"
 	"net"
 	"net/http"
@@ -28,6 +29,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/syncthing/syncthing/internal/gen/discosrv"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/stringutil"
 )
@@ -52,7 +54,7 @@ type apiSrv struct {
 }
 
 type replicator interface {
-	send(key *protocol.DeviceID, addrs []DatabaseAddress, seen int64)
+	send(key *protocol.DeviceID, addrs []*discosrv.DatabaseAddress, seen int64)
 }
 
 type requestID int64
@@ -65,7 +67,7 @@ type contextKey int
 
 const idKey contextKey = iota
 
-func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP, compression bool) *apiSrv {
+func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP, compression bool, desiredNotFoundRate float64) *apiSrv {
 	return &apiSrv{
 		addr:        addr,
 		cert:        cert,
@@ -76,13 +78,13 @@ func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator,
 		seenTracker: &retryAfterTracker{
 			name:         "seenTracker",
 			bucketStarts: time.Now(),
-			desiredRate:  250,
+			desiredRate:  desiredNotFoundRate / 2,
 			currentDelay: notFoundRetryUnknownMinSeconds,
 		},
 		notSeenTracker: &retryAfterTracker{
 			name:         "notSeenTracker",
 			bucketStarts: time.Now(),
-			desiredRate:  250,
+			desiredRate:  desiredNotFoundRate / 2,
 			currentDelay: notFoundRetryUnknownMaxSeconds / 2,
 		},
 	}
@@ -92,7 +94,7 @@ func (s *apiSrv) Serve(ctx context.Context) error {
 	if s.useHTTP {
 		listener, err := net.Listen("tcp", s.addr)
 		if err != nil {
-			log.Println("Listen:", err)
+			slog.ErrorContext(ctx, "Failed to listen", "error", err)
 			return err
 		}
 		s.listener = listener
@@ -106,7 +108,7 @@ func (s *apiSrv) Serve(ctx context.Context) error {
 
 		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
 		if err != nil {
-			log.Println("Listen:", err)
+			slog.ErrorContext(ctx, "Failed to listen", "error", err)
 			return err
 		}
 		s.listener = tlsListener
@@ -119,7 +121,9 @@ func (s *apiSrv) Serve(ctx context.Context) error {
 		ReadTimeout:    httpReadTimeout,
 		WriteTimeout:   httpWriteTimeout,
 		MaxHeaderBytes: httpMaxHeaderBytes,
-		ErrorLog:       log.New(io.Discard, "", 0),
+	}
+	if !debug {
+		srv.ErrorLog = log.New(io.Discard, "", 0)
 	}
 
 	go func() {
@@ -129,7 +133,7 @@ func (s *apiSrv) Serve(ctx context.Context) error {
 
 	err := srv.Serve(s.listener)
 	if err != nil {
-		log.Println("Serve:", err)
+		slog.ErrorContext(ctx, "Failed to serve", "error", err)
 	}
 	return err
 }
@@ -148,9 +152,7 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 	reqID := requestID(rand.Int63())
 	req = req.WithContext(context.WithValue(req.Context(), idKey, reqID))
 
-	if debug {
-		log.Println(reqID, req.Method, req.URL, req.Proto)
-	}
+	slog.Debug("Handling request", "id", reqID, "method", req.Method, "url", req.URL, "proto", req.Proto)
 
 	remoteAddr := &net.TCPAddr{
 		IP:   nil,
@@ -171,7 +173,7 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 		var err error
 		remoteAddr, err = net.ResolveTCPAddr("tcp", req.RemoteAddr)
 		if err != nil {
-			log.Println("remoteAddr:", err)
+			slog.Warn("Failed to resolve remote address", "address", req.RemoteAddr, "error", err)
 			lw.Header().Set("Retry-After", errorRetryAfterString())
 			http.Error(lw, "Internal Server Error", http.StatusInternalServerError)
 			apiRequestsTotal.WithLabelValues("no_remote_addr").Inc()
@@ -194,9 +196,7 @@ func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
 
 	deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
 	if err != nil {
-		if debug {
-			log.Println(reqID, "bad device param")
-		}
+		slog.Debug("Request with bad device param", "id", reqID, "error", err)
 		lookupRequestsTotal.WithLabelValues("bad_request").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
 		http.Error(w, "Bad Request", http.StatusBadRequest)
@@ -256,9 +256,7 @@ func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req
 
 	rawCert, err := certificateBytes(req)
 	if err != nil {
-		if debug {
-			log.Println(reqID, "no certificates:", err)
-		}
+		slog.Debug("Request without certificates", "id", reqID, "error", err)
 		announceRequestsTotal.WithLabelValues("no_certificate").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
 		http.Error(w, "Forbidden", http.StatusForbidden)
@@ -267,9 +265,7 @@ func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req
 
 	var ann announcement
 	if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
-		if debug {
-			log.Println(reqID, "decode:", err)
-		}
+		slog.Debug("Failed to decode request", "id", reqID, "error", err)
 		announceRequestsTotal.WithLabelValues("bad_request").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
 		http.Error(w, "Bad Request", http.StatusBadRequest)
@@ -280,6 +276,7 @@ func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req
 
 	addresses := fixupAddresses(remoteAddr, ann.Addresses)
 	if len(addresses) == 0 {
+		slog.Debug("Request without addresses", "id", reqID, "error", err)
 		announceRequestsTotal.WithLabelValues("bad_request").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
 		http.Error(w, "Bad Request", http.StatusBadRequest)
@@ -287,6 +284,7 @@ func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req
 	}
 
 	if err := s.handleAnnounce(deviceID, addresses); err != nil {
+		slog.Debug("Failed to handle request", "id", reqID, "error", err)
 		announceRequestsTotal.WithLabelValues("internal_error").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
@@ -297,6 +295,7 @@ func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req
 
 	w.Header().Set("Reannounce-After", reannounceAfterString())
 	w.WriteHeader(http.StatusNoContent)
+	slog.Debug("Device announced", "id", reqID, "device", deviceID, "addresses", addresses)
 }
 
 func (s *apiSrv) Stop() {
@@ -312,10 +311,12 @@ func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string)
 	slices.Sort(addresses)
 	addresses = slices.Compact(addresses)
 
-	dbAddrs := make([]DatabaseAddress, len(addresses))
+	dbAddrs := make([]*discosrv.DatabaseAddress, len(addresses))
 	for i := range addresses {
-		dbAddrs[i].Address = addresses[i]
-		dbAddrs[i].Expires = expire
+		dbAddrs[i] = &discosrv.DatabaseAddress{
+			Address: addresses[i],
+			Expires: expire,
+		}
 	}
 
 	seen := now.UnixNano()
@@ -326,7 +327,7 @@ func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string)
 }
 
 func handlePing(w http.ResponseWriter, _ *http.Request) {
-	w.WriteHeader(204)
+	w.WriteHeader(http.StatusNoContent)
 }
 
 func certificateBytes(req *http.Request) ([]byte, error) {
@@ -336,7 +337,7 @@ func certificateBytes(req *http.Request) ([]byte, error) {
 
 	var bs []byte
 
-	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
+	if hdr := req.Header.Get("X-Ssl-Cert"); hdr != "" {
 		if strings.Contains(hdr, "%") {
 			// Nginx using $ssl_client_escaped_cert
 			// The certificate is in PEM format with url encoding.
@@ -499,6 +500,7 @@ func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
 
 type loggingResponseWriter struct {
 	http.ResponseWriter
+
 	statusCode int
 }
 
@@ -511,7 +513,7 @@ func (lrw *loggingResponseWriter) WriteHeader(code int) {
 	lrw.ResponseWriter.WriteHeader(code)
 }
 
-func addressStrs(dbAddrs []DatabaseAddress) []string {
+func addressStrs(dbAddrs []*discosrv.DatabaseAddress) []string {
 	res := make([]string, len(dbAddrs))
 	for i, a := range dbAddrs {
 		res[i] = a.Address

cmd/stdiscosrv/apisrv_test.go

@@ -111,11 +111,11 @@ func BenchmarkAPIRequests(b *testing.B) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	go db.Serve(ctx)
-	api := newAPISrv("127.0.0.1:0", tls.Certificate{}, db, nil, true, true)
+	api := newAPISrv("127.0.0.1:0", tls.Certificate{}, db, nil, true, true, 1000)
 	srv := httptest.NewServer(http.HandlerFunc(api.handler))
 
 	kf := b.TempDir() + "/cert"
-	crt, err := tlsutil.NewCertificate(kf+".crt", kf+".key", "localhost", 7)
+	crt, err := tlsutil.NewCertificate(kf+".crt", kf+".key", "localhost", 7, true)
 	if err != nil {
 		b.Fatal(err)
 	}

cmd/stdiscosrv/database.go

@@ -4,9 +4,6 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//go:generate go run ../../proto/scripts/protofmt.go database.proto
-//go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
-
 package main
 
 import (
@@ -16,7 +13,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"io"
-	"log"
+	"log/slog"
 	"os"
 	"path"
 	"runtime"
@@ -25,7 +22,13 @@ import (
 	"time"
 
 	"github.com/puzpuzpuz/xsync/v3"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/syncthing/syncthing/internal/blob"
+	"github.com/syncthing/syncthing/internal/gen/discosrv"
+	"github.com/syncthing/syncthing/internal/protoutil"
 	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/rand"
 )
 
 type clock interface {
@@ -39,50 +42,61 @@ func (defaultClock) Now() time.Time {
 }
 
 type database interface {
-	put(key *protocol.DeviceID, rec DatabaseRecord) error
-	merge(key *protocol.DeviceID, addrs []DatabaseAddress, seen int64) error
-	get(key *protocol.DeviceID) (DatabaseRecord, error)
+	put(key *protocol.DeviceID, rec *discosrv.DatabaseRecord) error
+	merge(key *protocol.DeviceID, addrs []*discosrv.DatabaseAddress, seen int64) error
+	get(key *protocol.DeviceID) (*discosrv.DatabaseRecord, error)
 }
 
 type inMemoryStore struct {
-	m             *xsync.MapOf[protocol.DeviceID, DatabaseRecord]
+	m             *xsync.MapOf[protocol.DeviceID, *discosrv.DatabaseRecord]
 	dir           string
 	flushInterval time.Duration
-	s3            *s3Copier
+	blobs         blob.Store
+	objKey        string
 	clock         clock
 }
 
-func newInMemoryStore(dir string, flushInterval time.Duration, s3 *s3Copier) *inMemoryStore {
+func newInMemoryStore(dir string, flushInterval time.Duration, blobs blob.Store) *inMemoryStore {
+	hn, err := os.Hostname()
+	if err != nil {
+		hn = rand.String(8)
+	}
 	s := &inMemoryStore{
-		m:             xsync.NewMapOf[protocol.DeviceID, DatabaseRecord](),
+		m:             xsync.NewMapOf[protocol.DeviceID, *discosrv.DatabaseRecord](),
 		dir:           dir,
 		flushInterval: flushInterval,
-		s3:            s3,
+		blobs:         blobs,
+		objKey:        hn + ".db",
 		clock:         defaultClock{},
 	}
 	nr, err := s.read()
-	if os.IsNotExist(err) && s3 != nil {
-		// Try to read from AWS
-		fd, cerr := os.Create(path.Join(s.dir, "records.db"))
+	if os.IsNotExist(err) && blobs != nil {
+		// Try to read from blob storage
+		latestKey, cerr := blobs.LatestKey(context.Background())
 		if cerr != nil {
-			log.Println("Error creating database file:", err)
+			slog.Error("Failed to find database in blob storage", "error", cerr)
 			return s
 		}
-		if err := s3.downloadLatest(fd); err != nil {
-			log.Printf("Error reading database from S3: %v", err)
+		fd, cerr := os.Create(path.Join(s.dir, "records.db"))
+		if cerr != nil {
+			slog.Error("Failed to create database file", "error", cerr)
+			return s
+		}
+		if cerr := blobs.Download(context.Background(), latestKey, fd); cerr != nil {
+			slog.Error("Failed to download database from blob storage", "error", cerr)
 		}
 		_ = fd.Close()
 		nr, err = s.read()
 	}
 	if err != nil {
-		log.Println("Error reading database:", err)
+		slog.Error("Failed to read database", "error", err)
 	}
-	log.Printf("Read %d records from database", nr)
+	slog.Info("Loaded database", "records", nr)
 	s.expireAndCalculateStatistics()
 	return s
 }
 
-func (s *inMemoryStore) put(key *protocol.DeviceID, rec DatabaseRecord) error {
+func (s *inMemoryStore) put(key *protocol.DeviceID, rec *discosrv.DatabaseRecord) error {
 	t0 := time.Now()
 	s.m.Store(*key, rec)
 	databaseOperations.WithLabelValues(dbOpPut, dbResSuccess).Inc()
@@ -90,16 +104,17 @@ func (s *inMemoryStore) put(key *protocol.DeviceID, rec DatabaseRecord) error {
 	return nil
 }
 
-func (s *inMemoryStore) merge(key *protocol.DeviceID, addrs []DatabaseAddress, seen int64) error {
+func (s *inMemoryStore) merge(key *protocol.DeviceID, addrs []*discosrv.DatabaseAddress, seen int64) error {
 	t0 := time.Now()
 
-	newRec := DatabaseRecord{
+	newRec := &discosrv.DatabaseRecord{
 		Addresses: addrs,
 		Seen:      seen,
 	}
 
-	oldRec, _ := s.m.Load(*key)
-	newRec = merge(oldRec, newRec)
+	if oldRec, ok := s.m.Load(*key); ok {
+		newRec = merge(oldRec, newRec)
+	}
 	s.m.Store(*key, newRec)
 
 	databaseOperations.WithLabelValues(dbOpMerge, dbResSuccess).Inc()
@@ -108,7 +123,7 @@ func (s *inMemoryStore) merge(key *protocol.DeviceID, addrs []DatabaseAddress, s
 	return nil
 }
 
-func (s *inMemoryStore) get(key *protocol.DeviceID) (DatabaseRecord, error) {
+func (s *inMemoryStore) get(key *protocol.DeviceID) (*discosrv.DatabaseRecord, error) {
 	t0 := time.Now()
 	defer func() {
 		databaseOperationSeconds.WithLabelValues(dbOpGet).Observe(time.Since(t0).Seconds())
@@ -117,7 +132,7 @@ func (s *inMemoryStore) get(key *protocol.DeviceID) (DatabaseRecord, error) {
 	rec, ok := s.m.Load(*key)
 	if !ok {
 		databaseOperations.WithLabelValues(dbOpGet, dbResNotFound).Inc()
-		return DatabaseRecord{}, nil
+		return &discosrv.DatabaseRecord{}, nil
 	}
 
 	rec.Addresses = expire(rec.Addresses, s.clock.Now())
@@ -138,13 +153,13 @@ loop:
 	for {
 		select {
 		case <-t.C:
-			log.Println("Calculating statistics")
+			slog.InfoContext(ctx, "Calculating statistics")
 			s.expireAndCalculateStatistics()
-			log.Println("Flushing database")
+			slog.InfoContext(ctx, "Flushing database")
 			if err := s.write(); err != nil {
-				log.Println("Error writing database:", err)
+				slog.ErrorContext(ctx, "Failed to write database", "error", err)
 			}
-			log.Println("Finished flushing database")
+			slog.InfoContext(ctx, "Finished flushing database")
 			t.Reset(s.flushInterval)
 
 		case <-ctx.Done():
@@ -163,7 +178,7 @@ func (s *inMemoryStore) expireAndCalculateStatistics() {
 	current, currentIPv4, currentIPv6, currentIPv6GUA, last24h, last1w := 0, 0, 0, 0, 0, 0
 
 	n := 0
-	s.m.Range(func(key protocol.DeviceID, rec DatabaseRecord) bool {
+	s.m.Range(func(key protocol.DeviceID, rec *discosrv.DatabaseRecord) bool {
 		if n%1000 == 0 {
 			runtime.Gosched()
 		}
@@ -241,14 +256,14 @@ func (s *inMemoryStore) write() (err error) {
 	if err != nil {
 		return err
 	}
-	bw := bufio.NewWriter(fd)
+	bw := bufio.NewWriterSize(fd, 1<<20)
 
 	var buf []byte
 	var rangeErr error
 	now := s.clock.Now()
 	cutoff1w := now.Add(-7 * 24 * time.Hour).UnixNano()
 	n := 0
-	s.m.Range(func(key protocol.DeviceID, value DatabaseRecord) bool {
+	s.m.Range(func(key protocol.DeviceID, value *discosrv.DatabaseRecord) bool {
 		if n%1000 == 0 {
 			runtime.Gosched()
 		}
@@ -258,16 +273,16 @@ func (s *inMemoryStore) write() (err error) {
 			// drop the record if it's older than a week
 			return true
 		}
-		rec := ReplicationRecord{
+		rec := &discosrv.ReplicationRecord{
 			Key:       key[:],
 			Addresses: value.Addresses,
 			Seen:      value.Seen,
 		}
-		s := rec.Size()
+		s := proto.Size(rec)
 		if s+4 > len(buf) {
 			buf = make([]byte, s+4)
 		}
-		n, err := rec.MarshalTo(buf[4:])
+		n, err := protoutil.MarshalTo(buf[4:], rec)
 		if err != nil {
 			rangeErr = err
 			return false
@@ -295,18 +310,24 @@ func (s *inMemoryStore) write() (err error) {
 		return err
 	}
 
-	// Upload to S3
-	if s.s3 != nil {
+	if info, err := os.Lstat(dbf); err == nil {
+		slog.Info("Saved database", "name", dbf, "size", info.Size(), "modtime", info.ModTime())
+	} else {
+		slog.Warn("Failed to stat database after save", "error", err)
+	}
+
+	// Upload to blob storage
+	if s.blobs != nil {
 		fd, err = os.Open(dbf)
 		if err != nil {
-			log.Printf("Error uploading database to S3: %v", err)
+			slog.Error("Failed to upload database to blob storage", "error", err)
 			return nil
 		}
 		defer fd.Close()
-		if err := s.s3.upload(fd); err != nil {
-			log.Printf("Error uploading database to S3: %v", err)
+		if err := s.blobs.Upload(context.Background(), s.objKey, fd); err != nil {
+			slog.Error("Failed to upload database to blob storage", "error", err)
 		}
-		log.Println("Finished uploading database")
+		slog.Info("Finished uploading database")
 	}
 
 	return nil
@@ -336,8 +357,8 @@ func (s *inMemoryStore) read() (int, error) {
 		if _, err := io.ReadFull(br, buf[:n]); err != nil {
 			return nr, err
 		}
-		rec := ReplicationRecord{}
-		if err := rec.Unmarshal(buf[:n]); err != nil {
+		rec := &discosrv.ReplicationRecord{}
+		if err := proto.Unmarshal(buf[:n], rec); err != nil {
 			return nr, err
 		}
 		key, err := protocol.DeviceIDFromBytes(rec.Key)
@@ -345,13 +366,13 @@ func (s *inMemoryStore) read() (int, error) {
 			key, err = protocol.DeviceIDFromString(string(rec.Key))
 		}
 		if err != nil {
-			log.Println("Bad device ID:", err)
+			slog.Error("Got bad device ID while reading database", "error", err)
 			continue
 		}
 
-		slices.SortFunc(rec.Addresses, DatabaseAddress.Cmp)
-		rec.Addresses = slices.CompactFunc(rec.Addresses, DatabaseAddress.Equal)
-		s.m.Store(key, DatabaseRecord{
+		slices.SortFunc(rec.Addresses, Cmp)
+		rec.Addresses = slices.CompactFunc(rec.Addresses, Equal)
+		s.m.Store(key, &discosrv.DatabaseRecord{
 			Addresses: expire(rec.Addresses, s.clock.Now()),
 			Seen:      rec.Seen,
 		})
@@ -364,7 +385,7 @@ func (s *inMemoryStore) read() (int, error) {
 // result is the union of the two address sets, with the newer expiry time
 // chosen for any duplicates. The address list in a is overwritten and
 // reused for the result.
-func merge(a, b DatabaseRecord) DatabaseRecord {
+func merge(a, b *discosrv.DatabaseRecord) *discosrv.DatabaseRecord {
 	// Both lists must be sorted for this to work.
 
 	a.Seen = max(a.Seen, b.Seen)
@@ -383,7 +404,7 @@ func merge(a, b DatabaseRecord) DatabaseRecord {
 			aIdx++
 		case 1:
 			// a > b, insert b before a
-			a.Addresses = append(a.Addresses[:aIdx], append([]DatabaseAddress{b.Addresses[bIdx]}, a.Addresses[aIdx:]...)...)
+			a.Addresses = append(a.Addresses[:aIdx], append([]*discosrv.DatabaseAddress{b.Addresses[bIdx]}, a.Addresses[aIdx:]...)...)
 			bIdx++
 		}
 	}
@@ -397,7 +418,7 @@ func merge(a, b DatabaseRecord) DatabaseRecord {
 // expire returns the list of addresses after removing expired entries.
 // Expiration happen in place, so the slice given as the parameter is
 // destroyed. Internal order is preserved.
-func expire(addrs []DatabaseAddress, now time.Time) []DatabaseAddress {
+func expire(addrs []*discosrv.DatabaseAddress, now time.Time) []*discosrv.DatabaseAddress {
 	cutoff := now.UnixNano()
 	naddrs := addrs[:0]
 	for i := range addrs {
@@ -415,13 +436,13 @@ func expire(addrs []DatabaseAddress, now time.Time) []DatabaseAddress {
 	return naddrs
 }
 
-func (d DatabaseAddress) Cmp(other DatabaseAddress) (n int) {
+func Cmp(d, other *discosrv.DatabaseAddress) (n int) {
 	if c := cmp.Compare(d.Address, other.Address); c != 0 {
 		return c
 	}
 	return cmp.Compare(d.Expires, other.Expires)
 }
 
-func (d DatabaseAddress) Equal(other DatabaseAddress) bool {
+func Equal(d, other *discosrv.DatabaseAddress) bool {
 	return d.Address == other.Address
 }

cmd/stdiscosrv/database.pb.go

@@ -1,792 +0,0 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: database.proto
-
-package main
-
-import (
-	fmt "fmt"
-	_ "github.com/gogo/protobuf/gogoproto"
-	proto "github.com/gogo/protobuf/proto"
-	io "io"
-	math "math"
-	math_bits "math/bits"
-)
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
-
-type DatabaseRecord struct {
-	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
-	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
-}
-
-func (m *DatabaseRecord) Reset()         { *m = DatabaseRecord{} }
-func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
-func (*DatabaseRecord) ProtoMessage()    {}
-func (*DatabaseRecord) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b90fe3356ea5df07, []int{0}
-}
-func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
-	return m.Unmarshal(b)
-}
-func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	if deterministic {
-		return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
-	} else {
-		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
-		if err != nil {
-			return nil, err
-		}
-		return b[:n], nil
-	}
-}
-func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DatabaseRecord.Merge(m, src)
-}
-func (m *DatabaseRecord) XXX_Size() int {
-	return m.Size()
-}
-func (m *DatabaseRecord) XXX_DiscardUnknown() {
-	xxx_messageInfo_DatabaseRecord.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_DatabaseRecord proto.InternalMessageInfo
-
-type ReplicationRecord struct {
-	Key       []byte            `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
-	Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses"`
-	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
-}
-
-func (m *ReplicationRecord) Reset()         { *m = ReplicationRecord{} }
-func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
-func (*ReplicationRecord) ProtoMessage()    {}
-func (*ReplicationRecord) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b90fe3356ea5df07, []int{1}
-}
-func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
-	return m.Unmarshal(b)
-}
-func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	if deterministic {
-		return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
-	} else {
-		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
-		if err != nil {
-			return nil, err
-		}
-		return b[:n], nil
-	}
-}
-func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ReplicationRecord.Merge(m, src)
-}
-func (m *ReplicationRecord) XXX_Size() int {
-	return m.Size()
-}
-func (m *ReplicationRecord) XXX_DiscardUnknown() {
-	xxx_messageInfo_ReplicationRecord.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_ReplicationRecord proto.InternalMessageInfo
-
-type DatabaseAddress struct {
-	Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
-	Expires int64  `protobuf:"varint,2,opt,name=expires,proto3" json:"expires,omitempty"`
-}
-
-func (m *DatabaseAddress) Reset()         { *m = DatabaseAddress{} }
-func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
-func (*DatabaseAddress) ProtoMessage()    {}
-func (*DatabaseAddress) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b90fe3356ea5df07, []int{2}
-}
-func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
-	return m.Unmarshal(b)
-}
-func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	if deterministic {
-		return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
-	} else {
-		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
-		if err != nil {
-			return nil, err
-		}
-		return b[:n], nil
-	}
-}
-func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DatabaseAddress.Merge(m, src)
-}
-func (m *DatabaseAddress) XXX_Size() int {
-	return m.Size()
-}
-func (m *DatabaseAddress) XXX_DiscardUnknown() {
-	xxx_messageInfo_DatabaseAddress.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_DatabaseAddress proto.InternalMessageInfo
-
-func init() {
-	proto.RegisterType((*DatabaseRecord)(nil), "main.DatabaseRecord")
-	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
-	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
-}
-
-func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
-
-var fileDescriptor_b90fe3356ea5df07 = []byte{
-	// 243 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4b, 0x49, 0x2c, 0x49,
-	0x4c, 0x4a, 0x2c, 0x4e, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0xc9, 0x4d, 0xcc, 0xcc,
-	0x93, 0x52, 0x2e, 0x4a, 0x2d, 0xc8, 0x2f, 0xd6, 0x07, 0x0b, 0x25, 0x95, 0xa6, 0xe9, 0xa7, 0xe7,
-	0xa7, 0xe7, 0x83, 0x39, 0x60, 0x16, 0x44, 0xa9, 0x52, 0x3c, 0x17, 0x9f, 0x0b, 0x54, 0x73, 0x50,
-	0x6a, 0x72, 0x7e, 0x51, 0x8a, 0x90, 0x25, 0x17, 0x67, 0x62, 0x4a, 0x4a, 0x51, 0x6a, 0x71, 0x71,
-	0x6a, 0xb1, 0x04, 0xa3, 0x02, 0xb3, 0x06, 0xb7, 0x91, 0xa8, 0x1e, 0xc8, 0x40, 0x3d, 0x98, 0x42,
-	0x47, 0x88, 0xb4, 0x13, 0xcb, 0x89, 0x7b, 0xf2, 0x0c, 0x41, 0x08, 0xd5, 0x42, 0x42, 0x5c, 0x2c,
-	0xc5, 0xa9, 0xa9, 0x79, 0x12, 0xcc, 0x0a, 0x8c, 0x1a, 0xcc, 0x41, 0x60, 0xb6, 0x52, 0x09, 0x97,
-	0x60, 0x50, 0x6a, 0x41, 0x4e, 0x66, 0x72, 0x62, 0x49, 0x66, 0x7e, 0x1e, 0xd4, 0x0e, 0x01, 0x2e,
-	0xe6, 0xec, 0xd4, 0x4a, 0x09, 0x46, 0x05, 0x46, 0x0d, 0x9e, 0x20, 0x10, 0x13, 0xd5, 0x56, 0x26,
-	0x8a, 0x6d, 0x75, 0xe5, 0xe2, 0x47, 0xd3, 0x27, 0x24, 0xc1, 0xc5, 0x0e, 0xd5, 0x03, 0xb6, 0x97,
-	0x33, 0x08, 0xc6, 0x05, 0xc9, 0xa4, 0x56, 0x14, 0x64, 0x16, 0x81, 0x6d, 0x06, 0x99, 0x01, 0xe3,
-	0x3a, 0xc9, 0x9c, 0x78, 0x28, 0xc7, 0x70, 0xe2, 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f,
-	0x1e, 0xc9, 0x31, 0x4e, 0x78, 0x2c, 0xc7, 0x70, 0xe1, 0xb1, 0x1c, 0xc3, 0x8d, 0xc7, 0x72, 0x0c,
-	0x49, 0x6c, 0xe0, 0x20, 0x34, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0xc6, 0x0b, 0x9b, 0x77, 0x7f,
-	0x01, 0x00, 0x00,
-}
-
-func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if m.Seen != 0 {
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
-		i--
-		dAtA[i] = 0x18
-	}
-	if len(m.Addresses) > 0 {
-		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
-			{
-				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-				if err != nil {
-					return 0, err
-				}
-				i -= size
-				i = encodeVarintDatabase(dAtA, i, uint64(size))
-			}
-			i--
-			dAtA[i] = 0xa
-		}
-	}
-	return len(dAtA) - i, nil
-}
-
-func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if m.Seen != 0 {
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
-		i--
-		dAtA[i] = 0x18
-	}
-	if len(m.Addresses) > 0 {
-		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
-			{
-				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-				if err != nil {
-					return 0, err
-				}
-				i -= size
-				i = encodeVarintDatabase(dAtA, i, uint64(size))
-			}
-			i--
-			dAtA[i] = 0x12
-		}
-	}
-	if len(m.Key) > 0 {
-		i -= len(m.Key)
-		copy(dAtA[i:], m.Key)
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
-		i--
-		dAtA[i] = 0xa
-	}
-	return len(dAtA) - i, nil
-}
-
-func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if m.Expires != 0 {
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
-		i--
-		dAtA[i] = 0x10
-	}
-	if len(m.Address) > 0 {
-		i -= len(m.Address)
-		copy(dAtA[i:], m.Address)
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
-		i--
-		dAtA[i] = 0xa
-	}
-	return len(dAtA) - i, nil
-}
-
-func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
-	offset -= sovDatabase(v)
-	base := offset
-	for v >= 1<<7 {
-		dAtA[offset] = uint8(v&0x7f | 0x80)
-		v >>= 7
-		offset++
-	}
-	dAtA[offset] = uint8(v)
-	return base
-}
-func (m *DatabaseRecord) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	if len(m.Addresses) > 0 {
-		for _, e := range m.Addresses {
-			l = e.Size()
-			n += 1 + l + sovDatabase(uint64(l))
-		}
-	}
-	if m.Seen != 0 {
-		n += 1 + sovDatabase(uint64(m.Seen))
-	}
-	return n
-}
-
-func (m *ReplicationRecord) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	l = len(m.Key)
-	if l > 0 {
-		n += 1 + l + sovDatabase(uint64(l))
-	}
-	if len(m.Addresses) > 0 {
-		for _, e := range m.Addresses {
-			l = e.Size()
-			n += 1 + l + sovDatabase(uint64(l))
-		}
-	}
-	if m.Seen != 0 {
-		n += 1 + sovDatabase(uint64(m.Seen))
-	}
-	return n
-}
-
-func (m *DatabaseAddress) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	l = len(m.Address)
-	if l > 0 {
-		n += 1 + l + sovDatabase(uint64(l))
-	}
-	if m.Expires != 0 {
-		n += 1 + sovDatabase(uint64(m.Expires))
-	}
-	return n
-}
-
-func sovDatabase(x uint64) (n int) {
-	return (math_bits.Len64(x|1) + 6) / 7
-}
-func sozDatabase(x uint64) (n int) {
-	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
-}
-func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowDatabase
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: DatabaseRecord: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: DatabaseRecord: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Addresses = append(m.Addresses, DatabaseAddress{})
-			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		case 3:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
-			}
-			m.Seen = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Seen |= int64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		default:
-			iNdEx = preIndex
-			skippy, err := skipDatabase(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if (skippy < 0) || (iNdEx+skippy) < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowDatabase
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: ReplicationRecord: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: ReplicationRecord: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
-			}
-			var byteLen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				byteLen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if byteLen < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			postIndex := iNdEx + byteLen
-			if postIndex < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...)
-			if m.Key == nil {
-				m.Key = []byte{}
-			}
-			iNdEx = postIndex
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Addresses = append(m.Addresses, DatabaseAddress{})
-			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		case 3:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
-			}
-			m.Seen = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Seen |= int64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		default:
-			iNdEx = preIndex
-			skippy, err := skipDatabase(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if (skippy < 0) || (iNdEx+skippy) < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowDatabase
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: DatabaseAddress: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: DatabaseAddress: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Address", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Address = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		case 2:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Expires", wireType)
-			}
-			m.Expires = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Expires |= int64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		default:
-			iNdEx = preIndex
-			skippy, err := skipDatabase(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if (skippy < 0) || (iNdEx+skippy) < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-func skipDatabase(dAtA []byte) (n int, err error) {
-	l := len(dAtA)
-	iNdEx := 0
-	depth := 0
-	for iNdEx < l {
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return 0, ErrIntOverflowDatabase
-			}
-			if iNdEx >= l {
-				return 0, io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		wireType := int(wire & 0x7)
-		switch wireType {
-		case 0:
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return 0, ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return 0, io.ErrUnexpectedEOF
-				}
-				iNdEx++
-				if dAtA[iNdEx-1] < 0x80 {
-					break
-				}
-			}
-		case 1:
-			iNdEx += 8
-		case 2:
-			var length int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return 0, ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return 0, io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				length |= (int(b) & 0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if length < 0 {
-				return 0, ErrInvalidLengthDatabase
-			}
-			iNdEx += length
-		case 3:
-			depth++
-		case 4:
-			if depth == 0 {
-				return 0, ErrUnexpectedEndOfGroupDatabase
-			}
-			depth--
-		case 5:
-			iNdEx += 4
-		default:
-			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
-		}
-		if iNdEx < 0 {
-			return 0, ErrInvalidLengthDatabase
-		}
-		if depth == 0 {
-			return iNdEx, nil
-		}
-	}
-	return 0, io.ErrUnexpectedEOF
-}
-
-var (
-	ErrInvalidLengthDatabase        = fmt.Errorf("proto: negative length found during unmarshaling")
-	ErrIntOverflowDatabase          = fmt.Errorf("proto: integer overflow")
-	ErrUnexpectedEndOfGroupDatabase = fmt.Errorf("proto: unexpected end of group")
-)

cmd/stdiscosrv/database.proto

@@ -1,32 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-syntax = "proto3";
-
-package main;
-
-import "repos/protobuf/gogoproto/gogo.proto";
-
-option (gogoproto.goproto_getters_all) = false;
-option (gogoproto.goproto_unkeyed_all) = false;
-option (gogoproto.goproto_unrecognized_all) = false;
-option (gogoproto.goproto_sizecache_all) = false;
-
-message DatabaseRecord {
-    repeated DatabaseAddress addresses = 1 [(gogoproto.nullable) = false];
-    int64                    seen      = 3; // Unix nanos, last device announce
-}
-
-message ReplicationRecord {
-    bytes                    key       = 1; // raw 32 byte device ID
-    repeated DatabaseAddress addresses = 2 [(gogoproto.nullable) = false];
-    int64                    seen      = 3; // Unix nanos, last device announce
-}
-
-message DatabaseAddress {
-    string address = 1;
-    int64  expires = 2; // Unix nanos
-}

cmd/stdiscosrv/database_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/syncthing/syncthing/internal/gen/discosrv"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -39,7 +40,7 @@ func TestDatabaseGetSet(t *testing.T) {
 
 	// Put a record
 
-	rec.Addresses = []DatabaseAddress{
+	rec.Addresses = []*discosrv.DatabaseAddress{
 		{Address: "tcp://1.2.3.4:5", Expires: tc.Now().Add(time.Minute).UnixNano()},
 	}
 	if err := db.put(&protocol.EmptyDeviceID, rec); err != nil {
@@ -65,7 +66,7 @@ func TestDatabaseGetSet(t *testing.T) {
 
 	tc.wind(30 * time.Second)
 
-	addrs := []DatabaseAddress{
+	addrs := []*discosrv.DatabaseAddress{
 		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
 	}
 	if err := db.merge(&protocol.EmptyDeviceID, addrs, tc.Now().UnixNano()); err != nil {
@@ -112,7 +113,7 @@ func TestDatabaseGetSet(t *testing.T) {
 
 	// Set an address
 
-	addrs = []DatabaseAddress{
+	addrs = []*discosrv.DatabaseAddress{
 		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
 	}
 	if err := db.merge(&protocol.GlobalDeviceID, addrs, tc.Now().UnixNano()); err != nil {
@@ -134,28 +135,28 @@ func TestDatabaseGetSet(t *testing.T) {
 func TestFilter(t *testing.T) {
 	// all cases are expired with t=10
 	cases := []struct {
-		a []DatabaseAddress
-		b []DatabaseAddress
+		a []*discosrv.DatabaseAddress
+		b []*discosrv.DatabaseAddress
 	}{
 		{
 			a: nil,
 			b: nil,
 		},
 		{
-			a: []DatabaseAddress{{Address: "a", Expires: 9}, {Address: "b", Expires: 9}, {Address: "c", Expires: 9}},
-			b: []DatabaseAddress{},
+			a: []*discosrv.DatabaseAddress{{Address: "a", Expires: 9}, {Address: "b", Expires: 9}, {Address: "c", Expires: 9}},
+			b: []*discosrv.DatabaseAddress{},
 		},
 		{
-			a: []DatabaseAddress{{Address: "a", Expires: 10}},
-			b: []DatabaseAddress{{Address: "a", Expires: 10}},
+			a: []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
+			b: []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
 		},
 		{
-			a: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
-			b: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
+			a: []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
+			b: []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
 		},
 		{
-			a: []DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
-			b: []DatabaseAddress{{Address: "b", Expires: 15}, {Address: "d", Expires: 15}},
+			a: []*discosrv.DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
+			b: []*discosrv.DatabaseAddress{{Address: "b", Expires: 15}, {Address: "d", Expires: 15}},
 		},
 	}
 
@@ -169,62 +170,62 @@ func TestFilter(t *testing.T) {
 
 func TestMerge(t *testing.T) {
 	cases := []struct {
-		a, b, res []DatabaseAddress
+		a, b, res []*discosrv.DatabaseAddress
 	}{
 		{nil, nil, nil},
 		{
 			nil,
-			[]DatabaseAddress{{Address: "a", Expires: 10}},
-			[]DatabaseAddress{{Address: "a", Expires: 10}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
 		},
 		{
 			nil,
-			[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
-			[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
 		},
 		{
-			[]DatabaseAddress{{Address: "a", Expires: 10}},
-			[]DatabaseAddress{{Address: "a", Expires: 15}},
-			[]DatabaseAddress{{Address: "a", Expires: 15}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 15}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 15}},
 		},
 		{
-			[]DatabaseAddress{{Address: "a", Expires: 10}},
-			[]DatabaseAddress{{Address: "b", Expires: 15}},
-			[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
+			[]*discosrv.DatabaseAddress{{Address: "b", Expires: 15}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
 		},
 		{
-			[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
-			[]DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 15}},
-			[]DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 15}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 15}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 15}},
 		},
 		{
-			[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
-			[]DatabaseAddress{{Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
-			[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
+			[]*discosrv.DatabaseAddress{{Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
 		},
 		{
-			[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
-			[]DatabaseAddress{{Address: "b", Expires: 5}, {Address: "c", Expires: 20}},
-			[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
+			[]*discosrv.DatabaseAddress{{Address: "b", Expires: 5}, {Address: "c", Expires: 20}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
 		},
 		{
-			[]DatabaseAddress{{Address: "y", Expires: 10}, {Address: "z", Expires: 10}},
-			[]DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}},
-			[]DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "y", Expires: 10}, {Address: "z", Expires: 10}},
+			[]*discosrv.DatabaseAddress{{Address: "y", Expires: 10}, {Address: "z", Expires: 10}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "y", Expires: 10}, {Address: "z", Expires: 10}},
 		},
 		{
-			[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "d", Expires: 10}},
-			[]DatabaseAddress{{Address: "b", Expires: 5}, {Address: "c", Expires: 20}},
-			[]DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}, {Address: "d", Expires: 10}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "d", Expires: 10}},
+			[]*discosrv.DatabaseAddress{{Address: "b", Expires: 5}, {Address: "c", Expires: 20}},
+			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}, {Address: "d", Expires: 10}},
 		},
 	}
 
 	for _, tc := range cases {
-		rec := merge(DatabaseRecord{Addresses: tc.a}, DatabaseRecord{Addresses: tc.b})
+		rec := merge(&discosrv.DatabaseRecord{Addresses: tc.a}, &discosrv.DatabaseRecord{Addresses: tc.b})
 		if fmt.Sprint(rec.Addresses) != fmt.Sprint(tc.res) {
 			t.Errorf("Incorrect result %v, expected %v", rec.Addresses, tc.res)
 		}
-		rec = merge(DatabaseRecord{Addresses: tc.b}, DatabaseRecord{Addresses: tc.a})
+		rec = merge(&discosrv.DatabaseRecord{Addresses: tc.b}, &discosrv.DatabaseRecord{Addresses: tc.a})
 		if fmt.Sprint(rec.Addresses) != fmt.Sprint(tc.res) {
 			t.Errorf("Incorrect result %v, expected %v", rec.Addresses, tc.res)
 		}
@@ -233,9 +234,9 @@ func TestMerge(t *testing.T) {
 
 func BenchmarkMergeEqual(b *testing.B) {
 	for i := 0; i < b.N; i++ {
-		ar := []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}}
-		br := []DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 10}}
-		res := merge(DatabaseRecord{Addresses: ar}, DatabaseRecord{Addresses: br})
+		ar := []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}}
+		br := []*discosrv.DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 10}}
+		res := merge(&discosrv.DatabaseRecord{Addresses: ar}, &discosrv.DatabaseRecord{Addresses: br})
 		if len(res.Addresses) != 2 {
 			b.Fatal("wrong length")
 		}

cmd/stdiscosrv/main.go

@@ -9,28 +9,30 @@ package main
 import (
 	"context"
 	"crypto/tls"
-	"log"
+	"log/slog"
 	"net/http"
 	"os"
 	"os/signal"
 	"runtime"
 	"time"
 
-	_ "net/http/pprof"
-
 	"github.com/alecthomas/kong"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/thejerf/suture/v4"
+
+	"github.com/syncthing/syncthing/internal/blob"
+	"github.com/syncthing/syncthing/internal/blob/azureblob"
+	"github.com/syncthing/syncthing/internal/blob/s3"
+	"github.com/syncthing/syncthing/internal/slogutil"
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/tlsutil"
-	"github.com/thejerf/suture/v4"
 )
 
 const (
-	addressExpiryTime          = 2 * time.Hour
-	databaseStatisticsInterval = 5 * time.Minute
+	addressExpiryTime = 2 * time.Hour
 
 	// Reannounce-After is set to reannounceAfterSeconds +
 	// random(reannounzeFuzzSeconds), similar for Retry-After
@@ -57,12 +59,13 @@ const (
 var debug = false
 
 type CLI struct {
-	Cert          string `group:"Listen" help:"Certificate file" default:"./cert.pem" env:"DISCOVERY_CERT_FILE"`
-	Key           string `group:"Listen" help:"Key file" default:"./key.pem" env:"DISCOVERY_KEY_FILE"`
-	HTTP          bool   `group:"Listen" help:"Listen on HTTP (behind an HTTPS proxy)" env:"DISCOVERY_HTTP"`
-	Compression   bool   `group:"Listen" help:"Enable GZIP compression of responses" env:"DISCOVERY_COMPRESSION"`
-	Listen        string `group:"Listen" help:"Listen address" default:":8443" env:"DISCOVERY_LISTEN"`
-	MetricsListen string `group:"Listen" help:"Metrics listen address" env:"DISCOVERY_METRICS_LISTEN"`
+	Cert                string  `group:"Listen" help:"Certificate file" default:"./cert.pem" env:"DISCOVERY_CERT_FILE"`
+	Key                 string  `group:"Listen" help:"Key file" default:"./key.pem" env:"DISCOVERY_KEY_FILE"`
+	HTTP                bool    `group:"Listen" help:"Listen on HTTP (behind an HTTPS proxy)" env:"DISCOVERY_HTTP"`
+	Compression         bool    `group:"Listen" help:"Enable GZIP compression of responses" env:"DISCOVERY_COMPRESSION"`
+	Listen              string  `group:"Listen" help:"Listen address" default:":8443" env:"DISCOVERY_LISTEN"`
+	MetricsListen       string  `group:"Listen" help:"Metrics listen address" env:"DISCOVERY_METRICS_LISTEN"`
+	DesiredNotFoundRate float64 `group:"Listen" help:"Desired maximum rate of not-found replies (/s)" default:"1000"`
 
 	DBDir           string        `group:"Database" help:"Database directory" default:"." env:"DISCOVERY_DB_DIR"`
 	DBFlushInterval time.Duration `group:"Database" help:"Interval between database flushes" default:"5m" env:"DISCOVERY_DB_FLUSH_INTERVAL"`
@@ -73,6 +76,10 @@ type CLI struct {
 	DBS3AccessKeyID string `name:"db-s3-access-key-id" group:"Database (S3 backup)" hidden:"true" help:"S3 access key ID for database" env:"DISCOVERY_DB_S3_ACCESS_KEY_ID"`
 	DBS3SecretKey   string `name:"db-s3-secret-key" group:"Database (S3 backup)" hidden:"true" help:"S3 secret key for database" env:"DISCOVERY_DB_S3_SECRET_KEY"`
 
+	DBAzureBlobAccount   string `name:"db-azure-blob-account" env:"DISCOVERY_DB_AZUREBLOB_ACCOUNT"`
+	DBAzureBlobKey       string `name:"db-azure-blob-key" env:"DISCOVERY_DB_AZUREBLOB_KEY"`
+	DBAzureBlobContainer string `name:"db-azure-blob-container" env:"DISCOVERY_DB_AZUREBLOB_CONTAINER"`
+
 	AMQPAddress string `group:"AMQP replication" hidden:"true" help:"Address to AMQP broker" env:"DISCOVERY_AMQP_ADDRESS"`
 
 	Debug   bool `short:"d" help:"Print debug output" env:"DISCOVERY_DEBUG"`
@@ -80,13 +87,16 @@ type CLI struct {
 }
 
 func main() {
-	log.SetOutput(os.Stdout)
-
 	var cli CLI
 	kong.Parse(&cli)
-	debug = cli.Debug
 
-	log.Println(build.LongVersionFor("stdiscosrv"))
+	level := slog.LevelInfo
+	if cli.Debug {
+		level = slog.LevelDebug
+	}
+	slogutil.SetDefaultLevel(level)
+
+	slog.Info(build.LongVersionFor("stdiscosrv"))
 	if cli.Version {
 		return
 	}
@@ -98,16 +108,18 @@ func main() {
 		var err error
 		cert, err = tls.LoadX509KeyPair(cli.Cert, cli.Key)
 		if os.IsNotExist(err) {
-			log.Println("Failed to load keypair. Generating one, this might take a while...")
-			cert, err = tlsutil.NewCertificate(cli.Cert, cli.Key, "stdiscosrv", 20*365)
+			slog.Info("Failed to load keypair. Generating one, this might take a while...")
+			cert, err = tlsutil.NewCertificate(cli.Cert, cli.Key, "stdiscosrv", 20*365, false)
 			if err != nil {
-				log.Fatalln("Failed to generate X509 key pair:", err)
+				slog.Error("Failed to generate X509 key pair", "error", err)
+				os.Exit(1)
 			}
 		} else if err != nil {
-			log.Fatalln("Failed to load keypair:", err)
+			slog.Error("Failed to load keypair", "error", err)
+			os.Exit(1)
 		}
 		devID := protocol.NewDeviceID(cert.Certificate[0])
-		log.Println("Server device ID is", devID)
+		slog.Info("Loaded certificate keypair", "deviceID", devID)
 	}
 
 	// Root of the service tree.
@@ -116,19 +128,21 @@ func main() {
 		Timeout:           2 * time.Minute,
 	})
 
-	// If configured, use S3 for database backups.
-	var s3c *s3Copier
+	// If configured, use blob storage for database backups.
+	var blobs blob.Store
+	var err error
 	if cli.DBS3Endpoint != "" {
-		hostname, err := os.Hostname()
-		if err != nil {
-			log.Fatalf("Failed to get hostname: %v", err)
-		}
-		key := hostname + ".db"
-		s3c = newS3Copier(cli.DBS3Endpoint, cli.DBS3Region, cli.DBS3Bucket, key, cli.DBS3AccessKeyID, cli.DBS3SecretKey)
+		blobs, err = s3.NewSession(cli.DBS3Endpoint, cli.DBS3Region, cli.DBS3Bucket, cli.DBS3AccessKeyID, cli.DBS3SecretKey)
+	} else if cli.DBAzureBlobAccount != "" {
+		blobs, err = azureblob.NewBlobStore(cli.DBAzureBlobAccount, cli.DBAzureBlobKey, cli.DBAzureBlobContainer)
+	}
+	if err != nil {
+		slog.Error("Failed to create blob store", "error", err)
+		os.Exit(1)
 	}
 
 	// Start the database.
-	db := newInMemoryStore(cli.DBDir, cli.DBFlushInterval, s3c)
+	db := newInMemoryStore(cli.DBDir, cli.DBFlushInterval, blobs)
 	main.Add(db)
 
 	// If we have an AMQP broker for replication, start that
@@ -141,7 +155,7 @@ func main() {
 	}
 
 	// Start the main API server.
-	qs := newAPISrv(cli.Listen, cert, db, repl, cli.HTTP, cli.Compression)
+	qs := newAPISrv(cli.Listen, cert, db, repl, cli.HTTP, cli.Compression, cli.DesiredNotFoundRate)
 	main.Add(qs)
 
 	// If we have a metrics port configured, start a metrics handler.
@@ -149,7 +163,9 @@ func main() {
 		go func() {
 			mux := http.NewServeMux()
 			mux.Handle("/metrics", promhttp.Handler())
-			log.Fatal(http.ListenAndServe(cli.MetricsListen, mux))
+			err := http.ListenAndServe(cli.MetricsListen, mux)
+			slog.Error("Failed to serve", "error", err)
+			os.Exit(1)
 		}()
 	}
 
@@ -161,7 +177,7 @@ func main() {
 	signal.Notify(signalChan, os.Interrupt)
 	go func() {
 		sig := <-signalChan
-		log.Printf("Received signal %s; shutting down", sig)
+		slog.Info("Received signal; shutting down", "signal", sig)
 		cancel()
 	}()
 

cmd/stdiscosrv/s3.go

@@ -1,97 +0,0 @@
-// Copyright (C) 2024 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"io"
-	"log"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-)
-
-type s3Copier struct {
-	endpoint    string
-	region      string
-	bucket      string
-	key         string
-	accessKeyID string
-	secretKey   string
-}
-
-func newS3Copier(endpoint, region, bucket, key, accessKeyID, secretKey string) *s3Copier {
-	return &s3Copier{
-		endpoint:    endpoint,
-		region:      region,
-		bucket:      bucket,
-		key:         key,
-		accessKeyID: accessKeyID,
-		secretKey:   secretKey,
-	}
-}
-
-func (s *s3Copier) upload(r io.Reader) error {
-	sess, err := session.NewSession(&aws.Config{
-		Region:      aws.String(s.region),
-		Endpoint:    aws.String(s.endpoint),
-		Credentials: credentials.NewStaticCredentials(s.accessKeyID, s.secretKey, ""),
-	})
-	if err != nil {
-		return err
-	}
-
-	uploader := s3manager.NewUploader(sess)
-	_, err = uploader.Upload(&s3manager.UploadInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(s.key),
-		Body:   r,
-	})
-	return err
-}
-
-func (s *s3Copier) downloadLatest(w io.WriterAt) error {
-	sess, err := session.NewSession(&aws.Config{
-		Region:      aws.String(s.region),
-		Endpoint:    aws.String(s.endpoint),
-		Credentials: credentials.NewStaticCredentials(s.accessKeyID, s.secretKey, ""),
-	})
-	if err != nil {
-		return err
-	}
-
-	svc := s3.New(sess)
-	resp, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(s.bucket)})
-	if err != nil {
-		return err
-	}
-
-	var lastKey string
-	var lastModified time.Time
-	var lastSize int64
-	for _, item := range resp.Contents {
-		if item.LastModified.After(lastModified) && *item.Size > lastSize {
-			lastKey = *item.Key
-			lastModified = *item.LastModified
-			lastSize = *item.Size
-		} else if lastModified.Sub(*item.LastModified) < 5*time.Minute && *item.Size > lastSize {
-			lastKey = *item.Key
-			lastSize = *item.Size
-		}
-	}
-
-	log.Println("Downloading database from", lastKey)
-	downloader := s3manager.NewDownloader(sess)
-	_, err = downloader.Download(w, &s3.GetObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(lastKey),
-	})
-	return err
-}

cmd/strelaysrv/listener.go

@@ -12,9 +12,8 @@ import (
 	"time"
 
 	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/tlsutil"
-
 	"github.com/syncthing/syncthing/lib/relay/protocol"
+	"github.com/syncthing/syncthing/lib/tlsutil"
 )
 
 var (
@@ -185,7 +184,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token strin
 					continue
 				}
 				// requestedPeer is the server, id is the client
-				ses := newSession(requestedPeer, id, sessionLimiter, globalLimiter)
+				ses := newSession(requestedPeer, id, sessionLimitBps, globalLimiter)
 
 				go ses.Serve()
 

cmd/strelaysrv/main.go

@@ -14,11 +14,14 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"syscall"
 	"time"
 
+	"golang.org/x/time/rate"
+
 	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/config"
@@ -30,7 +33,6 @@ import (
 	"github.com/syncthing/syncthing/lib/relay/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	_ "github.com/syncthing/syncthing/lib/upnp"
-	"golang.org/x/time/rate"
 )
 
 var (
@@ -50,7 +52,6 @@ var (
 	globalLimitBps    int
 	overLimit         atomic.Bool
 	descriptorLimit   int64
-	sessionLimiter    *rate.Limiter
 	globalLimiter     *rate.Limiter
 	networkBufferSize int
 
@@ -157,7 +158,7 @@ func main() {
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 20*365)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 20*365, false)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
@@ -227,9 +228,6 @@ func main() {
 		}
 	}
 
-	if sessionLimitBps > 0 {
-		sessionLimiter = rate.NewLimiter(rate.Limit(sessionLimitBps), 2*sessionLimitBps)
-	}
 	if globalLimitBps > 0 {
 		globalLimiter = rate.NewLimiter(rate.Limit(globalLimitBps), 2*globalLimitBps)
 	}
@@ -250,10 +248,10 @@ func main() {
 	query.Set("pingInterval", pingInterval.String())
 	query.Set("networkTimeout", networkTimeout.String())
 	if sessionLimitBps > 0 {
-		query.Set("sessionLimitBps", fmt.Sprint(sessionLimitBps))
+		query.Set("sessionLimitBps", strconv.Itoa(sessionLimitBps))
 	}
 	if globalLimitBps > 0 {
-		query.Set("globalLimitBps", fmt.Sprint(globalLimitBps))
+		query.Set("globalLimitBps", strconv.Itoa(globalLimitBps))
 	}
 	if statusAddr != "" {
 		query.Set("statusAddr", statusAddr)

cmd/strelaysrv/session.go

@@ -27,7 +27,7 @@ var (
 	bytesProxied    atomic.Int64
 )
 
-func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionRateLimit, globalRateLimit *rate.Limiter) *session {
+func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionLimitBps int, globalRateLimit *rate.Limiter) *session {
 	serverkey := make([]byte, 32)
 	_, err := rand.Read(serverkey)
 	if err != nil {
@@ -40,12 +40,17 @@ func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionRateLimit,
 		return nil
 	}
 
+	var sessionRateLimit *rate.Limiter
+	if sessionLimitBps > 0 {
+		sessionRateLimit = rate.NewLimiter(rate.Limit(sessionLimitBps), 2*sessionLimitBps)
+	}
 	ses := &session{
 		serverkey: serverkey,
 		serverid:  serverid,
 		clientkey: clientkey,
 		clientid:  clientid,
 		rateLimit: makeRateLimitFunc(sessionRateLimit, globalRateLimit),
+		limiter:   sessionRateLimit,
 		connsChan: make(chan net.Conn),
 		conns:     make([]net.Conn, 0, 2),
 	}
@@ -68,7 +73,6 @@ func findSession(key string) *session {
 	ses, ok := pendingSessions[key]
 	if !ok {
 		return nil
-
 	}
 	delete(pendingSessions, key)
 	return ses
@@ -110,6 +114,7 @@ type session struct {
 	clientid  syncthingprotocol.DeviceID
 
 	rateLimit func(bytes int)
+	limiter   *rate.Limiter
 
 	connsChan chan net.Conn
 	conns     []net.Conn

cmd/strelaysrv/status.go

@@ -122,7 +122,7 @@ func (r *rateCalculator) updateRates(interval time.Duration) {
 
 func (r *rateCalculator) rate(periods int) int64 {
 	var tot int64
-	for i := 0; i < periods; i++ {
+	for i := range periods {
 		tot += r.rates[i]
 	}
 	return tot / int64(periods)

cmd/strelaysrv/testutil/main.go

@@ -6,6 +6,7 @@ import (
 	"bufio"
 	"context"
 	"crypto/tls"
+	"errors"
 	"flag"
 	"log"
 	"net"
@@ -133,7 +134,8 @@ func connectToStdio(stdin <-chan string, conn net.Conn) {
 		conn.SetReadDeadline(time.Now().Add(time.Millisecond))
 		n, err := conn.Read(buf[0:])
 		if err != nil {
-			nerr, ok := err.(net.Error)
+			var nerr net.Error
+			ok := errors.As(err, &nerr)
 			if !ok || !nerr.Timeout() {
 				log.Println(err)
 				return

cmd/strelaysrv/utils.go

@@ -10,7 +10,7 @@ import (
 func setTCPOptions(conn net.Conn) error {
 	tcpConn, ok := conn.(*net.TCPConn)
 	if !ok {
-		return errors.New("Not a TCP connection")
+		return errors.New("not a TCP connection")
 	}
 	if err := tcpConn.SetLinger(0); err != nil {
 		return err

cmd/syncthing/blockprof.go

@@ -8,11 +8,14 @@ package main
 
 import (
 	"fmt"
+	"log/slog"
 	"os"
 	"runtime"
 	"runtime/pprof"
 	"syscall"
 	"time"
+
+	"github.com/syncthing/syncthing/internal/slogutil"
 )
 
 func startBlockProfiler() {
@@ -20,10 +23,10 @@ func startBlockProfiler() {
 	if profiler == nil {
 		panic("Couldn't find block profiler")
 	}
-	l.Debugln("Starting block profiling")
+	slog.Debug("Starting block profiling")
 	go func() {
 		err := saveBlockingProfiles(profiler) // Only returns on error
-		l.Warnln("Block profiler failed:", err)
+		slog.Error("Block profiler failed", slogutil.Error(err))
 		panic("Block profiler failed")
 	}()
 }

cmd/syncthing/cli/client.go

@@ -32,6 +32,7 @@ type APIClient interface {
 
 type apiClient struct {
 	http.Client
+
 	cfg    config.GUIConfiguration
 	apikey string
 }
@@ -91,11 +92,11 @@ func loadGUIConfig() (config.GUIConfiguration, error) {
 	guiCfg := cfg.GUI()
 
 	if guiCfg.Address() == "" {
-		return config.GUIConfiguration{}, errors.New("Could not find GUI Address")
+		return config.GUIConfiguration{}, errors.New("could not find GUI Address")
 	}
 
 	if guiCfg.APIKey == "" {
-		return config.GUIConfiguration{}, errors.New("Could not find GUI API key")
+		return config.GUIConfiguration{}, errors.New("could not find GUI API key")
 	}
 
 	return guiCfg, nil
@@ -113,7 +114,7 @@ func (c *apiClient) Endpoint() string {
 }
 
 func (c *apiClient) Do(req *http.Request) (*http.Response, error) {
-	req.Header.Set("X-API-Key", c.apikey)
+	req.Header.Set("X-Api-Key", c.apikey)
 	resp, err := c.Client.Do(req)
 	if err != nil {
 		return nil, err

cmd/syncthing/cli/config.go

@@ -10,6 +10,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net/http"
 	"reflect"
 
 	"github.com/AudriusButkevicius/recli"
@@ -18,6 +19,53 @@ import (
 	"github.com/urfave/cli"
 )
 
+// Try to mimic the kong output format through custom help templates
+var customAppHelpTemplate = `Usage: {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}}{{if .Commands}} <command> [flags]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}
+
+{{.Description}}{{if .VisibleFlags}}
+
+Flags:
+  {{range $index, $option := .VisibleFlags}}{{if $index}}
+  {{end}}{{$option}}{{end}}{{end}}{{if .VisibleCommands}}
+
+Commands:{{range .VisibleCategories}}{{if .Name}}
+
+  {{.Name}}:{{range .VisibleCommands}}
+    {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}{{else}}{{range .VisibleCommands}}
+  {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}{{end}}{{end}}{{end}}
+`
+
+var customCommandHelpTemplate = `Usage: {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}}{{if .VisibleFlags}} [flags]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}
+
+{{.Usage}}{{if .VisibleFlags}}
+
+Flags:
+  {{range $index, $option := .VisibleFlags}}{{if $index}}
+  {{end}}{{$option}}{{end}}{{end}}{{if .Category}}
+
+Category:
+  {{.Category}}{{end}}{{if .Description}}
+
+{{.Description}}{{end}}
+`
+
+var customSubcommandHelpTemplate = `Usage: {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} <command>{{if .VisibleFlags}} [flags]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}{{if .Description}}
+
+{{.Description}}{{else}}{{if .Usage}}
+
+{{.Usage}}{{end}}{{end}}{{if .VisibleFlags}}
+
+Flags:
+  {{range $index, $option := .VisibleFlags}}{{if $index}}
+  {{end}}{{$option}}{{end}}{{end}}{{if .VisibleCommands}}
+
+Commands:{{range .VisibleCategories}}{{if .Name}}
+
+  {{.Name}}:{{range .VisibleCommands}}
+    {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}{{else}}{{range .VisibleCommands}}
+  {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}{{end}}{{end}}{{end}}
+`
+
 type configHandler struct {
 	original, cfg config.Configuration
 	client        APIClient
@@ -28,13 +76,18 @@ type configCommand struct {
 	Args []string `arg:"" default:"-h"`
 }
 
-func (c *configCommand) Run(ctx Context, _ *kong.Context) error {
+func (c *configCommand) Run(ctx Context, outerCtx *kong.Context) error {
 	app := cli.NewApp()
-	app.Name = "syncthing"
-	app.Author = "The Syncthing Authors"
+	app.Name = "syncthing cli config"
+	app.HelpName = "syncthing cli config"
+	app.Description = outerCtx.Selected().Help
 	app.Metadata = map[string]interface{}{
 		"clientFactory": ctx.clientFactory,
 	}
+	app.CustomAppHelpTemplate = customAppHelpTemplate
+	// Override global templates, as this is out only usage of the package
+	cli.CommandHelpTemplate = customCommandHelpTemplate
+	cli.SubcommandHelpTemplate = customSubcommandHelpTemplate
 
 	h := new(configHandler)
 	h.client, h.err = ctx.clientFactory.getClient()
@@ -55,6 +108,8 @@ func (c *configCommand) Run(ctx Context, _ *kong.Context) error {
 
 	app.Commands = commands
 	app.HideHelp = true
+	// Explicitly re-add help only as flags, not as commands
+	app.Flags = []cli.Flag{cli.HelpFlag}
 	app.Before = h.configBefore
 	app.After = h.configAfter
 
@@ -86,7 +141,7 @@ func (h *configHandler) configAfter(_ *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		body, err := responseToBArray(resp)
 		if err != nil {
 			return err

cmd/syncthing/cli/debug.go

@@ -41,5 +41,4 @@ func (p *profileCommand) Run(ctx Context) error {
 type debugCommand struct {
 	File    fileCommand    `cmd:"" help:"Show information about a file (or directory/symlink)"`
 	Profile profileCommand `cmd:"" help:"Save a profile to help figuring out what Syncthing does"`
-	Index   indexCommand   `cmd:"" help:"Show information about the index (database)"`
 }

cmd/syncthing/cli/errors.go

@@ -9,6 +9,7 @@ package cli
 import (
 	"errors"
 	"fmt"
+	"net/http"
 	"strings"
 
 	"github.com/alecthomas/kong"
@@ -34,7 +35,7 @@ func (e *errorsPushCommand) Run(ctx Context) error {
 	if err != nil {
 		return err
 	}
-	if response.StatusCode != 200 {
+	if response.StatusCode != http.StatusOK {
 		errStr = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
 		bytes, err := responseToBArray(response)
 		if err != nil {

cmd/syncthing/cli/index.go

@@ -1,32 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cli
-
-import (
-	"github.com/alecthomas/kong"
-)
-
-type indexCommand struct {
-	Dump     struct{} `cmd:"" help:"Print the entire db"`
-	DumpSize struct{} `cmd:"" help:"Print the db size of different categories of information"`
-	Check    struct{} `cmd:"" help:"Check the database for inconsistencies"`
-	Account  struct{} `cmd:"" help:"Print key and value size statistics per key type"`
-}
-
-func (*indexCommand) Run(kongCtx *kong.Context) error {
-	switch kongCtx.Selected().Name {
-	case "dump":
-		return indexDump()
-	case "dump-size":
-		return indexDumpSize()
-	case "check":
-		return indexCheck()
-	case "account":
-		return indexAccount()
-	}
-	return nil
-}

cmd/syncthing/cli/index_accounting.go

@@ -1,62 +0,0 @@
-// Copyright (C) 2020 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cli
-
-import (
-	"fmt"
-	"os"
-	"text/tabwriter"
-)
-
-// indexAccount prints key and data size statistics per class
-func indexAccount() error {
-	ldb, err := getDB()
-	if err != nil {
-		return err
-	}
-
-	it, err := ldb.NewPrefixIterator(nil)
-	if err != nil {
-		return err
-	}
-
-	var ksizes [256]int
-	var dsizes [256]int
-	var counts [256]int
-	var max [256]int
-
-	for it.Next() {
-		key := it.Key()
-		t := key[0]
-		ds := len(it.Value())
-		ks := len(key)
-		s := ks + ds
-
-		counts[t]++
-		ksizes[t] += ks
-		dsizes[t] += ds
-		if s > max[t] {
-			max[t] = s
-		}
-	}
-
-	tw := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', tabwriter.AlignRight)
-	toti, totds, totks := 0, 0, 0
-	for t := range ksizes {
-		if ksizes[t] > 0 {
-			// yes metric kilobytes 🤘
-			fmt.Fprintf(tw, "0x%02x:\t%d items,\t%d KB keys +\t%d KB data,\t%d B +\t%d B avg,\t%d B max\t\n", t, counts[t], ksizes[t]/1000, dsizes[t]/1000, ksizes[t]/counts[t], dsizes[t]/counts[t], max[t])
-			toti += counts[t]
-			totds += dsizes[t]
-			totks += ksizes[t]
-		}
-	}
-	fmt.Fprintf(tw, "Total\t%d items,\t%d KB keys +\t%d KB data.\t\n", toti, totks/1000, totds/1000)
-	tw.Flush()
-
-	return nil
-}

cmd/syncthing/cli/index_dump.go

@@ -1,158 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cli
-
-import (
-	"encoding/binary"
-	"fmt"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-func indexDump() error {
-	ldb, err := getDB()
-	if err != nil {
-		return err
-	}
-	it, err := ldb.NewPrefixIterator(nil)
-	if err != nil {
-		return err
-	}
-	for it.Next() {
-		key := it.Key()
-		switch key[0] {
-		case db.KeyTypeDevice:
-			folder := binary.BigEndian.Uint32(key[1:])
-			device := binary.BigEndian.Uint32(key[1+4:])
-			name := nulString(key[1+4+4:])
-			fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
-
-			var f protocol.FileInfo
-			err := f.Unmarshal(it.Value())
-			if err != nil {
-				return err
-			}
-			fmt.Printf(" V:%v\n", f)
-
-		case db.KeyTypeGlobal:
-			folder := binary.BigEndian.Uint32(key[1:])
-			name := nulString(key[1+4:])
-			var flv db.VersionList
-			flv.Unmarshal(it.Value())
-			fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
-
-		case db.KeyTypeBlock:
-			folder := binary.BigEndian.Uint32(key[1:])
-			hash := key[1+4 : 1+4+32]
-			name := nulString(key[1+4+32:])
-			fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
-
-		case db.KeyTypeDeviceStatistic:
-			fmt.Printf("[dstat] K:%x V:%x\n", key, it.Value())
-
-		case db.KeyTypeFolderStatistic:
-			fmt.Printf("[fstat] K:%x V:%x\n", key, it.Value())
-
-		case db.KeyTypeVirtualMtime:
-			folder := binary.BigEndian.Uint32(key[1:])
-			name := nulString(key[1+4:])
-			val := it.Value()
-			var realTime, virtualTime time.Time
-			realTime.UnmarshalBinary(val[:len(val)/2])
-			virtualTime.UnmarshalBinary(val[len(val)/2:])
-			fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, realTime, virtualTime)
-
-		case db.KeyTypeFolderIdx:
-			key := binary.BigEndian.Uint32(key[1:])
-			fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
-
-		case db.KeyTypeDeviceIdx:
-			key := binary.BigEndian.Uint32(key[1:])
-			val := it.Value()
-			device := "<nil>"
-			if len(val) > 0 {
-				dev, err := protocol.DeviceIDFromBytes(val)
-				if err != nil {
-					device = fmt.Sprintf("<invalid %d bytes>", len(val))
-				} else {
-					device = dev.String()
-				}
-			}
-			fmt.Printf("[deviceidx] K:%d V:%s\n", key, device)
-
-		case db.KeyTypeIndexID:
-			device := binary.BigEndian.Uint32(key[1:])
-			folder := binary.BigEndian.Uint32(key[5:])
-			fmt.Printf("[indexid] D:%d F:%d I:%x\n", device, folder, it.Value())
-
-		case db.KeyTypeFolderMeta:
-			folder := binary.BigEndian.Uint32(key[1:])
-			fmt.Printf("[foldermeta] F:%d", folder)
-			var cs db.CountsSet
-			if err := cs.Unmarshal(it.Value()); err != nil {
-				fmt.Printf(" (invalid)\n")
-			} else {
-				fmt.Printf(" V:%v\n", cs)
-			}
-
-		case db.KeyTypeMiscData:
-			fmt.Printf("[miscdata] K:%q V:%q\n", key[1:], it.Value())
-
-		case db.KeyTypeSequence:
-			folder := binary.BigEndian.Uint32(key[1:])
-			seq := binary.BigEndian.Uint64(key[5:])
-			fmt.Printf("[sequence] F:%d S:%d V:%q\n", folder, seq, it.Value())
-
-		case db.KeyTypeNeed:
-			folder := binary.BigEndian.Uint32(key[1:])
-			file := string(key[5:])
-			fmt.Printf("[need] F:%d V:%q\n", folder, file)
-
-		case db.KeyTypeBlockList:
-			fmt.Printf("[blocklist] H:%x\n", key[1:])
-
-		case db.KeyTypeBlockListMap:
-			folder := binary.BigEndian.Uint32(key[1:])
-			hash := key[5:37]
-			fileName := string(key[37:])
-			fmt.Printf("[blocklistmap] F:%d H:%x N:%s\n", folder, hash, fileName)
-
-		case db.KeyTypeVersion:
-			fmt.Printf("[version] H:%x", key[1:])
-			var v protocol.Vector
-			err := v.Unmarshal(it.Value())
-			if err != nil {
-				fmt.Printf(" (invalid)\n")
-			} else {
-				fmt.Printf(" V:%v\n", v)
-			}
-
-		case db.KeyTypePendingFolder:
-			device := binary.BigEndian.Uint32(key[1:])
-			folder := string(key[5:])
-			var of db.ObservedFolder
-			of.Unmarshal(it.Value())
-			fmt.Printf("[pendingFolder] D:%d F:%s V:%v\n", device, folder, of)
-
-		case db.KeyTypePendingDevice:
-			device := "<invalid>"
-			dev, err := protocol.DeviceIDFromBytes(key[1:])
-			if err == nil {
-				device = dev.String()
-			}
-			var od db.ObservedDevice
-			od.Unmarshal(it.Value())
-			fmt.Printf("[pendingDevice] D:%v V:%v\n", device, od)
-
-		default:
-			fmt.Printf("[??? %d]\n  %x\n  %x\n", key[0], key, it.Value())
-		}
-	}
-	return nil
-}

cmd/syncthing/cli/index_dumpsize.go

@@ -1,88 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cli
-
-import (
-	"encoding/binary"
-	"fmt"
-	"sort"
-
-	"github.com/syncthing/syncthing/lib/db"
-)
-
-func indexDumpSize() error {
-	type sizedElement struct {
-		key  string
-		size int
-	}
-
-	ldb, err := getDB()
-	if err != nil {
-		return err
-	}
-
-	it, err := ldb.NewPrefixIterator(nil)
-	if err != nil {
-		return err
-	}
-
-	var elems []sizedElement
-	for it.Next() {
-		var ele sizedElement
-
-		key := it.Key()
-		switch key[0] {
-		case db.KeyTypeDevice:
-			folder := binary.BigEndian.Uint32(key[1:])
-			device := binary.BigEndian.Uint32(key[1+4:])
-			name := nulString(key[1+4+4:])
-			ele.key = fmt.Sprintf("DEVICE:%d:%d:%s", folder, device, name)
-
-		case db.KeyTypeGlobal:
-			folder := binary.BigEndian.Uint32(key[1:])
-			name := nulString(key[1+4:])
-			ele.key = fmt.Sprintf("GLOBAL:%d:%s", folder, name)
-
-		case db.KeyTypeBlock:
-			folder := binary.BigEndian.Uint32(key[1:])
-			hash := key[1+4 : 1+4+32]
-			name := nulString(key[1+4+32:])
-			ele.key = fmt.Sprintf("BLOCK:%d:%x:%s", folder, hash, name)
-
-		case db.KeyTypeDeviceStatistic:
-			ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])
-
-		case db.KeyTypeFolderStatistic:
-			ele.key = fmt.Sprintf("FOLDERSTATS:%s", key[1:])
-
-		case db.KeyTypeVirtualMtime:
-			ele.key = fmt.Sprintf("MTIME:%s", key[1:])
-
-		case db.KeyTypeFolderIdx:
-			id := binary.BigEndian.Uint32(key[1:])
-			ele.key = fmt.Sprintf("FOLDERIDX:%d", id)
-
-		case db.KeyTypeDeviceIdx:
-			id := binary.BigEndian.Uint32(key[1:])
-			ele.key = fmt.Sprintf("DEVICEIDX:%d", id)
-
-		default:
-			ele.key = fmt.Sprintf("UNKNOWN:%x", key)
-		}
-		ele.size = len(it.Value())
-		elems = append(elems, ele)
-	}
-
-	sort.Slice(elems, func(i, j int) bool {
-		return elems[i].size > elems[j].size
-	})
-	for _, ele := range elems {
-		fmt.Println(ele.key, ele.size)
-	}
-
-	return nil
-}

cmd/syncthing/cli/index_idxck.go

@@ -1,355 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cli
-
-import (
-	"bytes"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"sort"
-
-	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-type fileInfoKey struct {
-	folder uint32
-	device uint32
-	name   string
-}
-
-type globalKey struct {
-	folder uint32
-	name   string
-}
-
-type sequenceKey struct {
-	folder   uint32
-	sequence uint64
-}
-
-func indexCheck() (err error) {
-	ldb, err := getDB()
-	if err != nil {
-		return err
-	}
-
-	folders := make(map[uint32]string)
-	devices := make(map[uint32]string)
-	deviceToIDs := make(map[string]uint32)
-	fileInfos := make(map[fileInfoKey]protocol.FileInfo)
-	globals := make(map[globalKey]db.VersionList)
-	sequences := make(map[sequenceKey]string)
-	needs := make(map[globalKey]struct{})
-	blocklists := make(map[string]struct{})
-	versions := make(map[string]protocol.Vector)
-	usedBlocklists := make(map[string]struct{})
-	usedVersions := make(map[string]struct{})
-	var localDeviceKey uint32
-	success := true
-	defer func() {
-		if err == nil {
-			if success {
-				fmt.Println("Index check completed successfully.")
-			} else {
-				err = errors.New("Inconsistencies found in the index")
-			}
-		}
-	}()
-
-	it, err := ldb.NewPrefixIterator(nil)
-	if err != nil {
-		return err
-	}
-	for it.Next() {
-		key := it.Key()
-		switch key[0] {
-		case db.KeyTypeDevice:
-			folder := binary.BigEndian.Uint32(key[1:])
-			device := binary.BigEndian.Uint32(key[1+4:])
-			name := nulString(key[1+4+4:])
-
-			var f protocol.FileInfo
-			err := f.Unmarshal(it.Value())
-			if err != nil {
-				fmt.Println("Unable to unmarshal FileInfo:", err)
-				success = false
-				continue
-			}
-
-			fileInfos[fileInfoKey{folder, device, name}] = f
-
-		case db.KeyTypeGlobal:
-			folder := binary.BigEndian.Uint32(key[1:])
-			name := nulString(key[1+4:])
-			var flv db.VersionList
-			if err := flv.Unmarshal(it.Value()); err != nil {
-				fmt.Println("Unable to unmarshal VersionList:", err)
-				success = false
-				continue
-			}
-			globals[globalKey{folder, name}] = flv
-
-		case db.KeyTypeFolderIdx:
-			key := binary.BigEndian.Uint32(it.Key()[1:])
-			folders[key] = string(it.Value())
-
-		case db.KeyTypeDeviceIdx:
-			key := binary.BigEndian.Uint32(it.Key()[1:])
-			devices[key] = string(it.Value())
-			deviceToIDs[string(it.Value())] = key
-			if bytes.Equal(it.Value(), protocol.LocalDeviceID[:]) {
-				localDeviceKey = key
-			}
-
-		case db.KeyTypeSequence:
-			folder := binary.BigEndian.Uint32(key[1:])
-			seq := binary.BigEndian.Uint64(key[5:])
-			val := it.Value()
-			sequences[sequenceKey{folder, seq}] = string(val[9:])
-
-		case db.KeyTypeNeed:
-			folder := binary.BigEndian.Uint32(key[1:])
-			name := nulString(key[1+4:])
-			needs[globalKey{folder, name}] = struct{}{}
-
-		case db.KeyTypeBlockList:
-			hash := string(key[1:])
-			blocklists[hash] = struct{}{}
-
-		case db.KeyTypeVersion:
-			hash := string(key[1:])
-			var v protocol.Vector
-			if err := v.Unmarshal(it.Value()); err != nil {
-				fmt.Println("Unable to unmarshal Vector:", err)
-				success = false
-				continue
-			}
-			versions[hash] = v
-		}
-	}
-
-	if localDeviceKey == 0 {
-		fmt.Println("Missing key for local device in device index (bailing out)")
-		success = false
-		return
-	}
-
-	var missingSeq []sequenceKey
-	for fk, fi := range fileInfos {
-		if fk.name != fi.Name {
-			fmt.Printf("Mismatching FileInfo name, %q (key) != %q (actual)\n", fk.name, fi.Name)
-			success = false
-		}
-
-		folder := folders[fk.folder]
-		if folder == "" {
-			fmt.Printf("Unknown folder ID %d for FileInfo %q\n", fk.folder, fk.name)
-			success = false
-			continue
-		}
-		if devices[fk.device] == "" {
-			fmt.Printf("Unknown device ID %d for FileInfo %q, folder %q\n", fk.folder, fk.name, folder)
-			success = false
-		}
-
-		if fk.device == localDeviceKey {
-			sk := sequenceKey{fk.folder, uint64(fi.Sequence)}
-			name, ok := sequences[sk]
-			if !ok {
-				fmt.Printf("Sequence entry missing for FileInfo %q, folder %q, seq %d\n", fi.Name, folder, fi.Sequence)
-				missingSeq = append(missingSeq, sk)
-				success = false
-				continue
-			}
-			if name != fi.Name {
-				fmt.Printf("Sequence entry refers to wrong name, %q (seq) != %q (FileInfo), folder %q, seq %d\n", name, fi.Name, folder, fi.Sequence)
-				success = false
-			}
-		}
-
-		if len(fi.Blocks) == 0 && len(fi.BlocksHash) != 0 {
-			key := string(fi.BlocksHash)
-			if _, ok := blocklists[key]; !ok {
-				fmt.Printf("Missing block list for file %q, block list hash %x\n", fi.Name, fi.BlocksHash)
-				success = false
-			} else {
-				usedBlocklists[key] = struct{}{}
-			}
-		}
-
-		if fi.VersionHash != nil {
-			key := string(fi.VersionHash)
-			if _, ok := versions[key]; !ok {
-				fmt.Printf("Missing version vector for file %q, version hash %x\n", fi.Name, fi.VersionHash)
-				success = false
-			} else {
-				usedVersions[key] = struct{}{}
-			}
-		}
-
-		_, ok := globals[globalKey{fk.folder, fk.name}]
-		if !ok {
-			fmt.Printf("Missing global for file %q\n", fi.Name)
-			success = false
-			continue
-		}
-	}
-
-	// Aggregate the ranges of missing sequence entries, print them
-
-	sort.Slice(missingSeq, func(a, b int) bool {
-		if missingSeq[a].folder != missingSeq[b].folder {
-			return missingSeq[a].folder < missingSeq[b].folder
-		}
-		return missingSeq[a].sequence < missingSeq[b].sequence
-	})
-
-	var folder uint32
-	var startSeq, prevSeq uint64
-	for _, sk := range missingSeq {
-		if folder != sk.folder || sk.sequence != prevSeq+1 {
-			if folder != 0 {
-				fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
-			}
-			startSeq = sk.sequence
-			folder = sk.folder
-		}
-		prevSeq = sk.sequence
-	}
-	if folder != 0 {
-		fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
-	}
-
-	for gk, vl := range globals {
-		folder := folders[gk.folder]
-		if folder == "" {
-			fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
-			success = false
-		}
-		checkGlobal := func(i int, device []byte, version protocol.Vector, invalid, deleted bool) {
-			dev, ok := deviceToIDs[string(device)]
-			if !ok {
-				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, device)
-				success = false
-			}
-			fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
-			if !ok {
-				fmt.Printf("VersionList %q, folder %q, entry %d refers to unknown FileInfo\n", gk.name, folder, i)
-				success = false
-			}
-
-			fiv := fi.Version
-			if fi.VersionHash != nil {
-				fiv = versions[string(fi.VersionHash)]
-			}
-			if !fiv.Equal(version) {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, version, fi.Version)
-				success = false
-			}
-			if fi.IsInvalid() != invalid {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, invalid, fi.IsInvalid())
-				success = false
-			}
-			if fi.IsDeleted() != deleted {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo deleted mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, deleted, fi.IsDeleted())
-				success = false
-			}
-		}
-		for i, fv := range vl.RawVersions {
-			for _, device := range fv.Devices {
-				checkGlobal(i, device, fv.Version, false, fv.Deleted)
-			}
-			for _, device := range fv.InvalidDevices {
-				checkGlobal(i, device, fv.Version, true, fv.Deleted)
-			}
-		}
-
-		// If we need this file we should have a need entry for it. False
-		// positives from needsLocally for deleted files, where we might
-		// legitimately lack an entry if we never had it, and ignored files.
-		if needsLocally(vl) {
-			_, ok := needs[gk]
-			if !ok {
-				fv, _ := vl.GetGlobal()
-				devB, _ := fv.FirstDevice()
-				dev := deviceToIDs[string(devB)]
-				fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
-				if !fi.IsDeleted() && !fi.IsIgnored() {
-					fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
-				}
-			}
-		}
-	}
-
-	seenSeq := make(map[fileInfoKey]uint64)
-	for sk, name := range sequences {
-		folder := folders[sk.folder]
-		if folder == "" {
-			fmt.Printf("Unknown folder ID %d for sequence entry %d, %q\n", sk.folder, sk.sequence, name)
-			success = false
-			continue
-		}
-
-		if prev, ok := seenSeq[fileInfoKey{folder: sk.folder, name: name}]; ok {
-			fmt.Printf("Duplicate sequence entry for %q, folder %q, seq %d (prev %d)\n", name, folder, sk.sequence, prev)
-			success = false
-		}
-		seenSeq[fileInfoKey{folder: sk.folder, name: name}] = sk.sequence
-
-		fi, ok := fileInfos[fileInfoKey{sk.folder, localDeviceKey, name}]
-		if !ok {
-			fmt.Printf("Missing FileInfo for sequence entry %d, folder %q, %q\n", sk.sequence, folder, name)
-			success = false
-			continue
-		}
-		if fi.Sequence != int64(sk.sequence) {
-			fmt.Printf("Sequence mismatch for %q, folder %q, %d (key) != %d (FileInfo)\n", name, folder, sk.sequence, fi.Sequence)
-			success = false
-		}
-	}
-
-	for nk := range needs {
-		folder := folders[nk.folder]
-		if folder == "" {
-			fmt.Printf("Unknown folder ID %d for need entry %q\n", nk.folder, nk.name)
-			success = false
-			continue
-		}
-
-		vl, ok := globals[nk]
-		if !ok {
-			fmt.Printf("Missing global for need entry %q, folder %q\n", nk.name, folder)
-			success = false
-			continue
-		}
-
-		if !needsLocally(vl) {
-			fmt.Printf("Need entry for file we don't need, %q, folder %q\n", nk.name, folder)
-			success = false
-		}
-	}
-
-	if d := len(blocklists) - len(usedBlocklists); d > 0 {
-		fmt.Printf("%d block list entries out of %d needs GC\n", d, len(blocklists))
-	}
-	if d := len(versions) - len(usedVersions); d > 0 {
-		fmt.Printf("%d version entries out of %d needs GC\n", d, len(versions))
-	}
-
-	return nil
-}
-
-func needsLocally(vl db.VersionList) bool {
-	gfv, gok := vl.GetGlobal()
-	if !gok { // That's weird, but we hardly need something non-existent
-		return false
-	}
-	fv, ok := vl.Get(protocol.LocalDeviceID[:])
-	return db.Need(gfv, ok, fv.Version)
-}

cmd/syncthing/cli/main.go

@@ -14,15 +14,12 @@ import (
 	"github.com/alecthomas/kong"
 	"github.com/kballard/go-shellquote"
 
-	"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
 	"github.com/syncthing/syncthing/lib/config"
 )
 
 type CLI struct {
-	cmdutil.CommonOptions
-	DataDir    string `name:"data" placeholder:"PATH" env:"STDATADIR" help:"Set data directory (database and logs)"`
-	GUIAddress string `name:"gui-address"`
-	GUIAPIKey  string `name:"gui-apikey"`
+	GUIAddress string `name:"gui-address" env:"STGUIADDRESS"`
+	GUIAPIKey  string `name:"gui-apikey" env:"STGUIAPIKEY"`
 
 	Show       showCommand      `cmd:"" help:"Show command group"`
 	Debug      debugCommand     `cmd:"" help:"Debug command group"`
@@ -37,11 +34,6 @@ type Context struct {
 }
 
 func (cli CLI) AfterApply(kongCtx *kong.Context) error {
-	err := cmdutil.SetConfigDataLocationsFromFlags(cli.HomeDir, cli.ConfDir, cli.DataDir)
-	if err != nil {
-		return fmt.Errorf("command line options: %w", err)
-	}
-
 	clientFactory := &apiClientFactory{
 		cfg: config.GUIConfiguration{
 			RawAddress: cli.GUIAddress,

cmd/syncthing/cli/operations.go

@@ -10,6 +10,7 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
+	"net/http"
 	"path/filepath"
 
 	"github.com/alecthomas/kong"
@@ -63,7 +64,7 @@ func (f *folderOverrideCommand) Run(ctx Context) error {
 			if err != nil {
 				return err
 			}
-			if response.StatusCode != 200 {
+			if response.StatusCode != http.StatusOK {
 				errStr := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
 				bytes, err := responseToBArray(response)
 				if err != nil {

cmd/syncthing/cli/utils.go

@@ -17,8 +17,6 @@ import (
 	"path/filepath"
 
 	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/db/backend"
-	"github.com/syncthing/syncthing/lib/locations"
 )
 
 func responseToBArray(response *http.Response) ([]byte, error) {
@@ -133,19 +131,6 @@ func prettyPrintResponse(response *http.Response) error {
 	return prettyPrintJSON(data)
 }
 
-func getDB() (backend.Backend, error) {
-	return backend.OpenLevelDBRO(locations.Get(locations.Database))
-}
-
-func nulString(bs []byte) string {
-	for i := range bs {
-		if bs[i] == 0 {
-			return string(bs[:i])
-		}
-	}
-	return string(bs)
-}
-
 func normalizePath(path string) string {
 	return filepath.ToSlash(filepath.Clean(path))
 }

cmd/syncthing/cmdutil/options_common.go

@@ -1,16 +0,0 @@
-// Copyright (C) 2021 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cmdutil
-
-// CommonOptions are reused among several subcommands
-type CommonOptions struct {
-	buildCommonOptions
-	ConfDir         string `name:"config" placeholder:"PATH" env:"STCONFDIR" help:"Set configuration directory (config and keys)"`
-	HomeDir         string `name:"home" placeholder:"PATH" env:"STHOMEDIR" help:"Set configuration and data directory"`
-	NoDefaultFolder bool   `env:"STNODEFAULTFOLDER" help:"Don't create the \"default\" folder on first startup"`
-	SkipPortProbing bool   `help:"Don't try to find free ports for GUI and listen addresses on first startup"`
-}

cmd/syncthing/cmdutil/util.go

@@ -1,35 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cmdutil
-
-import (
-	"errors"
-
-	"github.com/syncthing/syncthing/lib/locations"
-)
-
-func SetConfigDataLocationsFromFlags(homeDir, confDir, dataDir string) error {
-	homeSet := homeDir != ""
-	confSet := confDir != ""
-	dataSet := dataDir != ""
-	switch {
-	case dataSet != confSet:
-		return errors.New("either both or none of --config and --data must be given, use --home to set both at once")
-	case homeSet && dataSet:
-		return errors.New("--home must not be used together with --config and --data")
-	case homeSet:
-		confDir = homeDir
-		dataDir = homeDir
-		fallthrough
-	case dataSet:
-		if err := locations.SetBaseDir(locations.ConfigBaseDir, confDir); err != nil {
-			return err
-		}
-		return locations.SetBaseDir(locations.DataBaseDir, dataDir)
-	}
-	return nil
-}

cmd/syncthing/crash_reporting.go

@@ -11,12 +11,15 @@ import (
 	"context"
 	"crypto/sha256"
 	"fmt"
+	"log/slog"
 	"net/http"
 	"os"
 	"path/filepath"
-	"sort"
+	"slices"
 	"strings"
 	"time"
+
+	"github.com/syncthing/syncthing/internal/slogutil"
 )
 
 const (
@@ -28,16 +31,18 @@ const (
 // directory to the crash reporting server as urlBase. Uploads are attempted
 // with the newest log first.
 //
-// This can can block for a long time. The context can set a final deadline
+// This can block for a long time. The context can set a final deadline
 // for this.
 func uploadPanicLogs(ctx context.Context, urlBase, dir string) {
 	files, err := filepath.Glob(filepath.Join(dir, "panic-*.log"))
 	if err != nil {
-		l.Warnln("Failed to list panic logs:", err)
+		slog.ErrorContext(ctx, "Failed to list panic logs", slogutil.Error(err))
 		return
 	}
 
-	sort.Sort(sort.Reverse(sort.StringSlice(files)))
+	slices.SortFunc(files, func(a, b string) int {
+		return strings.Compare(b, a)
+	})
 	for _, file := range files {
 		if strings.Contains(file, ".reported.") {
 			// We've already sent this file. It'll be cleaned out at some
@@ -46,7 +51,7 @@ func uploadPanicLogs(ctx context.Context, urlBase, dir string) {
 		}
 
 		if err := uploadPanicLog(ctx, urlBase, file); err != nil {
-			l.Warnln("Reporting crash:", err)
+			slog.ErrorContext(ctx, "Reporting crash", slogutil.Error(err))
 		} else {
 			// Rename the log so we don't have to try to report it again. This
 			// succeeds, or it does not. There is no point complaining about it.
@@ -69,7 +74,7 @@ func uploadPanicLog(ctx context.Context, urlBase, file string) error {
 	data = filterLogLines(data)
 
 	hash := fmt.Sprintf("%x", sha256.Sum256(data))
-	l.Infof("Reporting crash found in %s (report ID %s) ...\n", filepath.Base(file), hash[:8])
+	slog.InfoContext(ctx, "Reporting crash", slogutil.FilePath(filepath.Base(file)), slog.String("id", hash[:8]))
 
 	url := fmt.Sprintf("%s/%s", urlBase, hash)
 	headReq, err := http.NewRequest(http.MethodHead, url, nil)

cmd/syncthing/debug.go

@@ -6,10 +6,6 @@
 
 package main
 
-import (
-	"github.com/syncthing/syncthing/lib/logger"
-)
+import "github.com/syncthing/syncthing/internal/slogutil"
 
-var (
-	l = logger.DefaultLogger.NewFacility("main", "Main package")
-)
+func init() { slogutil.RegisterPackage("Main package") }

cmd/syncthing/decrypt/decrypt.go

@@ -17,6 +17,9 @@ import (
 	"os"
 	"path/filepath"
 
+	"google.golang.org/protobuf/proto"
+
+	"github.com/syncthing/syncthing/internal/gen/bep"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/osutil"
@@ -235,7 +238,7 @@ func (c *CLI) decryptFile(encFi *protocol.FileInfo, plainFi *protocol.FileInfo,
 		}
 
 		// Verify the hash against the plaintext block info
-		if !scanner.Validate(dec, plainBlock.Hash, 0) {
+		if !scanner.Validate(dec, plainBlock.Hash) {
 			// The block decrypted correctly but fails the hash check. This
 			// is odd and unexpected, but it it's still a valid block from
 			// the source. The file might have changed while we pulled it?
@@ -280,10 +283,11 @@ func loadEncryptedFileInfo(fd fs.File) (*protocol.FileInfo, error) {
 		return nil, err
 	}
 
-	var encFi protocol.FileInfo
-	if err := encFi.Unmarshal(trailer); err != nil {
+	var encFi bep.FileInfo
+	if err := proto.Unmarshal(trailer, &encFi); err != nil {
 		return nil, err
 	}
+	fi := protocol.FileInfoFromWire(&encFi)
 
-	return &encFi, nil
+	return &fi, nil
 }

cmd/syncthing/generate/generate.go

@@ -11,42 +11,25 @@ import (
 	"bufio"
 	"context"
 	"crypto/tls"
-	"errors"
 	"fmt"
+	"log/slog"
 	"os"
 
-	"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/locations"
-	"github.com/syncthing/syncthing/lib/logger"
-	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/syncthing"
 )
 
 type CLI struct {
-	cmdutil.CommonOptions
-	GUIUser     string `placeholder:"STRING" help:"Specify new GUI authentication user name"`
-	GUIPassword string `placeholder:"STRING" help:"Specify new GUI authentication password (use - to read from standard input)"`
+	GUIUser       string `placeholder:"STRING" help:"Specify new GUI authentication user name"`
+	GUIPassword   string `placeholder:"STRING" help:"Specify new GUI authentication password (use - to read from standard input)"`
+	NoPortProbing bool   `help:"Don't try to find free ports for GUI and listen addresses on first startup" env:"STNOPORTPROBING"`
 }
 
-func (c *CLI) Run(l logger.Logger) error {
-	if c.HideConsole {
-		osutil.HideConsole()
-	}
-
-	if c.HomeDir != "" {
-		if c.ConfDir != "" {
-			return errors.New("--home must not be used together with --config")
-		}
-		c.ConfDir = c.HomeDir
-	}
-	if c.ConfDir == "" {
-		c.ConfDir = locations.GetBaseDir(locations.ConfigBaseDir)
-	}
-
+func (c *CLI) Run() error {
 	// Support reading the password from a pipe or similar
 	if c.GUIPassword == "-" {
 		reader := bufio.NewReader(os.Stdin)
@@ -57,13 +40,13 @@ func (c *CLI) Run(l logger.Logger) error {
 		c.GUIPassword = string(password)
 	}
 
-	if err := Generate(l, c.ConfDir, c.GUIUser, c.GUIPassword, c.NoDefaultFolder, c.SkipPortProbing); err != nil {
+	if err := Generate(locations.GetBaseDir(locations.ConfigBaseDir), c.GUIUser, c.GUIPassword, c.NoPortProbing); err != nil {
 		return fmt.Errorf("failed to generate config and keys: %w", err)
 	}
 	return nil
 }
 
-func Generate(l logger.Logger, confDir, guiUser, guiPassword string, noDefaultFolder, skipPortProbing bool) error {
+func Generate(confDir, guiUser, guiPassword string, skipPortProbing bool) error {
 	dir, err := fs.ExpandTilde(confDir)
 	if err != nil {
 		return err
@@ -78,20 +61,21 @@ func Generate(l logger.Logger, confDir, guiUser, guiPassword string, noDefaultFo
 	certFile, keyFile := locations.Get(locations.CertFile), locations.Get(locations.KeyFile)
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err == nil {
-		l.Warnln("Key exists; will not overwrite.")
+		slog.Warn("Key exists; will not overwrite")
 	} else {
 		cert, err = syncthing.GenerateCertificate(certFile, keyFile)
 		if err != nil {
 			return fmt.Errorf("create certificate: %w", err)
 		}
 	}
+
 	myID = protocol.NewDeviceID(cert.Certificate[0])
-	l.Infoln("Device ID:", myID)
+	slog.Info("Calculated device ID", slog.String("device", myID.String()))
 
 	cfgFile := locations.Get(locations.ConfigFile)
 	cfg, _, err := config.Load(cfgFile, myID, events.NoopLogger)
 	if fs.IsNotExist(err) {
-		if cfg, err = syncthing.DefaultConfig(cfgFile, myID, events.NoopLogger, noDefaultFolder, skipPortProbing); err != nil {
+		if cfg, err = syncthing.DefaultConfig(cfgFile, myID, events.NoopLogger, skipPortProbing); err != nil {
 			return fmt.Errorf("create config: %w", err)
 		}
 	} else if err != nil {
@@ -104,7 +88,7 @@ func Generate(l logger.Logger, confDir, guiUser, guiPassword string, noDefaultFo
 
 	var updateErr error
 	waiter, err := cfg.Modify(func(cfg *config.Configuration) {
-		updateErr = updateGUIAuthentication(l, &cfg.GUI, guiUser, guiPassword)
+		updateErr = updateGUIAuthentication(&cfg.GUI, guiUser, guiPassword)
 	})
 	if err != nil {
 		return fmt.Errorf("modify config: %w", err)
@@ -120,17 +104,17 @@ func Generate(l logger.Logger, confDir, guiUser, guiPassword string, noDefaultFo
 	return nil
 }
 
-func updateGUIAuthentication(l logger.Logger, guiCfg *config.GUIConfiguration, guiUser, guiPassword string) error {
+func updateGUIAuthentication(guiCfg *config.GUIConfiguration, guiUser, guiPassword string) error {
 	if guiUser != "" && guiCfg.User != guiUser {
 		guiCfg.User = guiUser
-		l.Infoln("Updated GUI authentication user name:", guiUser)
+		slog.Info("Updated GUI authentication user", "name", guiUser)
 	}
 
 	if guiPassword != "" && guiCfg.Password != guiPassword {
 		if err := guiCfg.SetPassword(guiPassword); err != nil {
 			return fmt.Errorf("failed to set GUI authentication password: %w", err)
 		}
-		l.Infoln("Updated GUI authentication password.")
+		slog.Info("Updated GUI authentication password")
 	}
 	return nil
 }

cmd/syncthing/heapprof.go

@@ -8,18 +8,21 @@ package main
 
 import (
 	"fmt"
+	"log/slog"
 	"os"
 	"runtime"
 	"runtime/pprof"
 	"syscall"
 	"time"
+
+	"github.com/syncthing/syncthing/internal/slogutil"
 )
 
 func startHeapProfiler() {
-	l.Debugln("Starting heap profiling")
+	slog.Debug("Starting heap profiling")
 	go func() {
 		err := saveHeapProfiles(1) // Only returns on error
-		l.Warnln("Heap profiler failed:", err)
+		slog.Error("Heap profiler failed", slogutil.Error(err))
 		panic("Heap profiler failed")
 	}()
 }

cmd/syncthing/hideconsole_others.go

@@ -7,8 +7,8 @@
 //go:build !windows
 // +build !windows
 
-package cmdutil
+package main
 
-type buildCommonOptions struct {
+type buildSpecificOptions struct {
 	HideConsole bool `hidden:""`
 }

cmd/syncthing/hideconsole_windows.go

@@ -4,8 +4,8 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package cmdutil
+package main
 
-type buildCommonOptions struct {
-	HideConsole bool `name:"no-console" help:"Hide console window"`
+type buildSpecificOptions struct {
+	HideConsole bool `name:"no-console" help:"Hide console window" env:"STHIDECONSOLE"`
 }

cmd/syncthing/monitor.go

@@ -9,28 +9,31 @@ package main
 import (
 	"bufio"
 	"context"
+	"errors"
 	"fmt"
 	"io"
+	"log/slog"
 	"os"
 	"os/exec"
 	"os/signal"
 	"path/filepath"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/internal/slogutil"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/svcutil"
-	"github.com/syncthing/syncthing/lib/sync"
 )
 
 var (
 	stdoutFirstLines []string // The first 10 lines of stdout
 	stdoutLastLines  []string // The last 50 lines of stdout
-	stdoutMut        = sync.NewMutex()
+	stdoutMut        sync.Mutex
 )
 
 const (
@@ -43,9 +46,7 @@ const (
 	panicUploadNoticeWait = 10 * time.Second
 )
 
-func monitorMain(options serveOptions) {
-	l.SetPrefix("[monitor] ")
-
+func (c *serveCmd) monitorMain() {
 	var dst io.Writer = os.Stdout
 
 	logFile := locations.Get(locations.LogFile)
@@ -58,13 +59,13 @@ func monitorMain(options serveOptions) {
 		open := func(name string) (io.WriteCloser, error) {
 			return newAutoclosedFile(name, logFileAutoCloseDelay, logFileMaxOpenTime)
 		}
-		if options.LogMaxSize > 0 {
-			fileDst, err = newRotatedFile(logFile, open, int64(options.LogMaxSize), options.LogMaxFiles)
+		if c.LogMaxSize > 0 {
+			fileDst, err = newRotatedFile(logFile, open, int64(c.LogMaxSize), c.LogMaxFiles)
 		} else {
 			fileDst, err = open(logFile)
 		}
 		if err != nil {
-			l.Warnln("Failed to setup logging to file, proceeding with logging to stdout only:", err)
+			slog.Error("Failed to set up logging to file, proceeding with logging to stdout only", slogutil.Error(err))
 		} else {
 			if build.IsWindows {
 				// Translate line breaks to Windows standard
@@ -78,14 +79,14 @@ func monitorMain(options serveOptions) {
 			// Log to both stdout and file.
 			dst = io.MultiWriter(dst, fileDst)
 
-			l.Infof(`Log output saved to file "%s"`, logFile)
+			slog.Info("Saved log output", slogutil.FilePath(logFile))
 		}
 	}
 
 	args := os.Args
 	binary, err := getBinary(args[0])
 	if err != nil {
-		l.Warnln("Error starting the main Syncthing process:", err)
+		slog.Error("Failed to start the main Syncthing process", slogutil.Error(err))
 		panic("Error starting the main Syncthing process")
 	}
 	var restarts [restartCounts]time.Time
@@ -102,7 +103,7 @@ func monitorMain(options serveOptions) {
 		maybeReportPanics()
 
 		if t := time.Since(restarts[0]); t < restartLoopThreshold {
-			l.Warnf("%d restarts in %v; not retrying further", restartCounts, t)
+			slog.Error("Too many restarts; not retrying further", slog.Int("count", restartCounts), slog.Any("interval", t))
 			os.Exit(svcutil.ExitError.AsInt())
 		}
 
@@ -122,10 +123,10 @@ func monitorMain(options serveOptions) {
 			panic(err)
 		}
 
-		l.Debugln("Starting syncthing")
+		slog.Debug("Starting syncthing")
 		err = cmd.Start()
 		if err != nil {
-			l.Warnln("Error starting the main Syncthing process:", err)
+			slog.Error("Failed to start the main Syncthing process", slogutil.Error(err))
 			panic("Error starting the main Syncthing process")
 		}
 
@@ -134,7 +135,7 @@ func monitorMain(options serveOptions) {
 		stdoutLastLines = make([]string, 0, 50)
 		stdoutMut.Unlock()
 
-		wg := sync.NewWaitGroup()
+		var wg sync.WaitGroup
 
 		wg.Add(1)
 		go func() {
@@ -158,13 +159,13 @@ func monitorMain(options serveOptions) {
 		stopped := false
 		select {
 		case s := <-stopSign:
-			l.Infof("Signal %d received; exiting", s)
+			slog.Info("Received signal; exiting", "signal", s)
 			cmd.Process.Signal(sigTerm)
 			err = <-exit
 			stopped = true
 
 		case s := <-restartSign:
-			l.Infof("Signal %d received; restarting", s)
+			slog.Info("Received signal; restarting", "signal", s)
 			cmd.Process.Signal(sigHup)
 			err = <-exit
 
@@ -176,27 +177,28 @@ func monitorMain(options serveOptions) {
 			os.Exit(svcutil.ExitSuccess.AsInt())
 		}
 
-		if exiterr, ok := err.(*exec.ExitError); ok {
+		exiterr := &exec.ExitError{}
+		if errors.As(err, &exiterr) {
 			exitCode := exiterr.ExitCode()
-			if stopped || options.NoRestart {
+			if stopped || c.NoRestart {
 				os.Exit(exitCode)
 			}
 			if exitCode == svcutil.ExitUpgrade.AsInt() {
 				// Restart the monitor process to release the .old
 				// binary as part of the upgrade process.
-				l.Infoln("Restarting monitor...")
+				slog.Info("Restarting monitor...")
 				if err = restartMonitor(binary, args); err != nil {
-					l.Warnln("Restart:", err)
+					slog.Error("Failed to restart monitor", slogutil.Error(err))
 				}
 				os.Exit(exitCode)
 			}
 		}
 
-		if options.NoRestart {
+		if c.NoRestart {
 			os.Exit(svcutil.ExitError.AsInt())
 		}
 
-		l.Infoln("Syncthing exited:", err)
+		slog.Info("Syncthing exited", slogutil.Error(err))
 		time.Sleep(restartPause)
 
 		if first {
@@ -238,47 +240,29 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
 			return
 		}
 
-		if panicFd == nil {
-			dst.Write([]byte(line))
+		dst.Write([]byte(line))
 
-			if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
-				panicFd, err = os.Create(locations.GetTimestamped(locations.PanicLog))
-				if err != nil {
-					l.Warnln("Create panic log:", err)
-					continue
-				}
-
-				l.Warnf("Panic detected, writing to \"%s\"", panicFd.Name())
-				if strings.Contains(line, "leveldb") && strings.Contains(line, "corrupt") {
-					l.Warnln(`
-*********************************************************************************
-* Crash due to corrupt database.                                                *
-*                                                                               *
-* This crash usually occurs due to one of the following reasons:                *
-*  - Syncthing being stopped abruptly (killed/loss of power)                    *
-*  - Bad hardware (memory/disk issues)                                          *
-*  - Software that affects disk writes (SSD caching software and similar)       *
-*                                                                               *
-* Please see the following URL for instructions on how to recover:              *
-*   https://docs.syncthing.net/users/faq.html#my-syncthing-database-is-corrupt  *
-*********************************************************************************
-`)
-				} else {
-					l.Warnln("Please check for existing issues with similar panic message at https://github.com/syncthing/syncthing/issues/")
-					l.Warnln("If no issue with similar panic message exists, please create a new issue with the panic log attached")
-				}
-
-				stdoutMut.Lock()
-				for _, line := range stdoutFirstLines {
-					panicFd.WriteString(line)
-				}
-				panicFd.WriteString("...\n")
-				for _, line := range stdoutLastLines {
-					panicFd.WriteString(line)
-				}
-				stdoutMut.Unlock()
+		if panicFd == nil && (strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:")) {
+			panicFd, err = os.Create(locations.GetTimestamped(locations.PanicLog))
+			if err != nil {
+				slog.Error("Failed to create panic log", slogutil.Error(err))
+				continue
 			}
 
+			slog.Error("Panic detected, writing to file", slogutil.FilePath(panicFd.Name()))
+			slog.Info("Please check for existing issues with similar panic message at https://github.com/syncthing/syncthing/issues/")
+			slog.Info("If no issue with similar panic message exists, please create a new issue with the panic log attached")
+
+			stdoutMut.Lock()
+			for _, line := range stdoutFirstLines {
+				panicFd.WriteString(line)
+			}
+			panicFd.WriteString("...\n")
+			for _, line := range stdoutLastLines {
+				panicFd.WriteString(line)
+			}
+			stdoutMut.Unlock()
+
 			panicFd.WriteString("Panic at " + time.Now().Format(time.RFC3339) + "\n")
 		}
 
@@ -448,7 +432,6 @@ func newAutoclosedFile(name string, closeDelay, maxOpenTime time.Duration) (*aut
 		name:        name,
 		closeDelay:  closeDelay,
 		maxOpenTime: maxOpenTime,
-		mut:         sync.NewMutex(),
 		closed:      make(chan struct{}),
 		closeTimer:  time.NewTimer(time.Minute),
 	}
@@ -556,7 +539,7 @@ func maybeReportPanics() {
 	// Try to get a config to see if/where panics should be reported.
 	cfg, err := loadOrDefaultConfig()
 	if err != nil {
-		l.Warnln("Couldn't load config; not reporting crash")
+		slog.Error("Couldn't load config; not reporting crash")
 		return
 	}
 
@@ -576,7 +559,7 @@ func maybeReportPanics() {
 		case <-ctx.Done():
 			return
 		case <-time.After(panicUploadNoticeWait):
-			l.Warnln("Uploading crash reports is taking a while, please wait...")
+			slog.Warn("Uploading crash reports is taking a while, please wait")
 		}
 	}()
 

cmd/syncthing/monitor_test.go

@@ -140,7 +140,7 @@ func checkNotExist(t *testing.T, name string) {
 func TestAutoClosedFile(t *testing.T) {
 	os.RemoveAll("_autoclose")
 	defer os.RemoveAll("_autoclose")
-	os.Mkdir("_autoclose", 0755)
+	os.Mkdir("_autoclose", 0o755)
 	file := filepath.FromSlash("_autoclose/tmp")
 	data := []byte("hello, world\n")
 

cmd/syncthing/perfstats_unix.go

@@ -16,7 +16,11 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/locations"
+	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
+	"golang.org/x/exp/constraints"
 )
 
 func startPerfStats() {
@@ -29,37 +33,55 @@ func savePerfStats(file string) {
 		panic(err)
 	}
 
-	var prevUsage int64
-	var prevTime int64
-	var rusage syscall.Rusage
-	var memstats runtime.MemStats
+	var prevTime time.Time
+	var curRus, prevRus syscall.Rusage
+	var curMem, prevMem runtime.MemStats
 	var prevIn, prevOut int64
 
 	t0 := time.Now()
+	syscall.Getrusage(syscall.RUSAGE_SELF, &prevRus)
+	runtime.ReadMemStats(&prevMem)
+
+	fmt.Fprintf(fd, "TIME_S\tCPU_S\tHEAP_KIB\tRSS_KIB\tNETIN_KBPS\tNETOUT_KBPS\tDBSIZE_KIB\n")
+
 	for t := range time.NewTicker(250 * time.Millisecond).C {
-		if err := syscall.Getrusage(syscall.RUSAGE_SELF, &rusage); err != nil {
-			continue
-		}
-
-		curTime := time.Now().UnixNano()
-		timeDiff := curTime - prevTime
-		curUsage := rusage.Utime.Nano() + rusage.Stime.Nano()
-		usageDiff := curUsage - prevUsage
-		cpuUsagePercent := 100 * float64(usageDiff) / float64(timeDiff)
-		prevTime = curTime
-		prevUsage = curUsage
+		syscall.Getrusage(syscall.RUSAGE_SELF, &curRus)
+		runtime.ReadMemStats(&curMem)
 		in, out := protocol.TotalInOut()
-		var inRate, outRate float64
-		if timeDiff > 0 {
-			inRate = float64(in-prevIn) / (float64(timeDiff) / 1e9)    // bytes per second
-			outRate = float64(out-prevOut) / (float64(timeDiff) / 1e9) // bytes per second
+		timeDiff := t.Sub(prevTime)
+
+		rss := curRus.Maxrss
+		if build.IsDarwin {
+			rss /= 1024
 		}
+
+		fmt.Fprintf(fd, "%.03f\t%f\t%d\t%d\t%.0f\t%.0f\t%d\n",
+			t.Sub(t0).Seconds(),
+			rate(cpusec(&prevRus), cpusec(&curRus), timeDiff, 1),
+			(curMem.Sys-curMem.HeapReleased)/1024,
+			rss,
+			rate(prevIn, in, timeDiff, 1e3),
+			rate(prevOut, out, timeDiff, 1e3),
+			osutil.DirSize(locations.Get(locations.Database))/1024,
+		)
+
+		prevTime = t
+		prevRus = curRus
+		prevMem = curMem
 		prevIn, prevOut = in, out
-
-		runtime.ReadMemStats(&memstats)
-
-		startms := int(t.Sub(t0).Seconds() * 1000)
-
-		fmt.Fprintf(fd, "%d\t%f\t%d\t%d\t%.0f\t%.0f\n", startms, cpuUsagePercent, memstats.Alloc, memstats.Sys-memstats.HeapReleased, inRate, outRate)
 	}
 }
+
+func cpusec(r *syscall.Rusage) float64 {
+	return float64(r.Utime.Nano()+r.Stime.Nano()) / float64(time.Second)
+}
+
+type number interface {
+	constraints.Float | constraints.Integer
+}
+
+func rate[T number](prev, cur T, d time.Duration, div float64) float64 {
+	diff := cur - prev
+	rate := float64(diff) / d.Seconds() / div
+	return rate
+}

compat.yaml

@@ -26,3 +26,16 @@
     darwin: "20"
     linux: "2.6.32"
     windows: "10.0"
+
+- runtime: go1.24
+  requirements:
+    darwin: "20"
+    linux: "3.2"
+    windows: "10.0"
+
+- runtime: go1.25
+  requirements:
+    # macOS 12 (Monterey) per https://tip.golang.org/doc/go1.25#darwin
+    darwin: "21"
+    linux: "3.2"
+    windows: "10.0"

etc/linux-desktop/syncthing-start.desktop

@@ -2,7 +2,7 @@
 Name=Start Syncthing
 GenericName=File synchronization
 Comment=Starts the main syncthing process in the background.
-Exec=/usr/bin/syncthing serve --no-browser --logfile=default
+Exec=syncthing serve --no-browser --logfile=default
 Icon=syncthing
 Terminal=false
 Type=Application

etc/linux-desktop/syncthing-ui.desktop

@@ -2,7 +2,7 @@
 Name=Syncthing Web UI
 GenericName=File synchronization UI
 Comment=Opens Syncthing's Web UI in the default browser (Syncthing must already be started).
-Exec=/usr/bin/syncthing --browser-only
+Exec=syncthing browser
 Icon=syncthing
 Terminal=false
 Type=Application