build: Readd systemd files to deb releases (ref #7564) (#7899)

.deepsource.toml

@@ -1,7 +1,7 @@
 version = 1
 
-exclude_patterns = ["**/*.pb.go"]
-test_patterns = ["**/*_test.go"]
+exclude_patterns = ["*.pb.go"]
+test_patterns = ["*_test.go"]
 
 [[analyzers]]
 name = "go"

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+/AUTHORS    @calmh
+/*.md       @calmh

.github/ISSUE_TEMPLATE/01-feature.md

@@ -0,0 +1,13 @@
+---
+name: Feature request
+about: If you're just not sure how to do something, see "ask a question".
+labels: enhancement, needs-triage
+---
+
+### Include required information
+
+Please be sure to include at least:
+
+ - what problem your new feature would solve
+ - how or why you think it is generally useful (i.e., not just for you)
+ - what alternatives or workarounds you considered

.github/ISSUE_TEMPLATE/01-feature.yml

@@ -1,28 +0,0 @@
-name: Feature request
-description: File a new feature request
-labels: ["enhancement", "needs-triage"]
-body:
-
-  - type: textarea
-    id: feature
-    attributes:
-      label: Feature description
-      description: Please describe the behavior you'd like to see.
-    validations:
-      required: true
-
-  - type: textarea
-    id: problem-usecase
-    attributes:
-      label: Problem or use case
-      description: Please explain which problem this would solve, or what the use case is for the feature. Keep in mind that it's more likely to be implemented if it's generally useful for a larger number of users.
-    validations:
-      required: true
-
-  - type: textarea
-    id: alternatives
-    attributes:
-      label: Alternatives or workarounds
-      description: Please describe any alternatives or workarounds you have considered and, possibly, rejected.
-    validations:
-      required: true

.github/ISSUE_TEMPLATE/02-bug.md

@@ -0,0 +1,23 @@
+---
+name: Bug report
+about: If you're actually looking for support, see "ask a question".
+labels: bug, needs-triage
+---
+
+### Does your log mention database corruption?
+
+If your Syncthing log reports panics because of database corruption it is
+most likely a fault with your system's storage or memory. Affected log
+entries will contain lines starting with `panic: leveldb`. You will need to
+delete the index database to clear this, by running `syncthing
+-reset-database`.
+
+### Include required information
+
+Please be sure to include at least:
+
+ - which version of Syncthing and what operating system you are using
+ - browser and version, if applicable
+ - what happened,
+ - what you expected to happen instead, and
+ - any steps to reproduce the problem.

.github/ISSUE_TEMPLATE/02-bug.yml

@@ -1,51 +0,0 @@
-name: Bug report
-description: If you're actually looking for support instead, see "I need help / I have a question".
-labels: ["bug", "needs-triage"]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        :no_entry_sign: If you want to report a security issue, please see [our Security Policy](https://syncthing.net/security/) and do not report the issue here.
-
-        :interrobang: If you are not sure if there is a bug, but something isn't working right and you need help, please [use the forum](https://forum.syncthing.net/).
-
-  - type: textarea
-    id: what-happened
-    attributes:
-      label: What happened?
-      description: Also tell us, what did you expect to happen, and any steps we might use to reproduce the problem.
-      placeholder: Tell us what you see!
-    validations:
-      required: true
-
-  - type: input
-    id: version
-    attributes:
-      label: Syncthing version
-      description: What version of Syncthing are you running?
-      placeholder: v1.27.4
-    validations:
-      required: true
-
-  - type: input
-    id: platform
-    attributes:
-      label: Platform & operating system
-      description: On what platform(s) are you seeing the problem?
-      placeholder: Linux arm64
-    validations:
-      required: true
-
-  - type: input
-    id: browser
-    attributes:
-      label: Browser version
-      description: If the problem is related to the GUI, describe your browser and version.
-      placeholder: Safari 17.3.1
-
-  - type: textarea
-    id: logs
-    attributes:
-      label: Relevant log output
-      description: Please copy and paste any relevant log output or crash backtrace. This will be automatically formatted into code, so no need for backticks.
-      render: shell

.github/dependabot.yml

@@ -1,13 +1,7 @@
 version: 2
 updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: monthly
-    open-pull-requests-limit: 10
-
-  - package-ecosystem: "gomod"
-    directory: "/"
-    schedule:
-      interval: monthly
-    open-pull-requests-limit: 10
+- package-ecosystem: gomod
+  directory: "/"
+  schedule:
+    interval: weekly
+  open-pull-requests-limit: 10

.github/workflows/build-infra-dockers.yaml

@@ -1,85 +0,0 @@
-name: Build Infrastructure Images
-
-on:
-  push:
-    branches:
-      - infrastructure
-      - infra-*
-
-env:
-  GO_VERSION: "~1.23.0"
-  CGO_ENABLED: "0"
-  BUILD_USER: docker
-  BUILD_HOST: github.syncthing.net
-
-permissions:
-  contents: read
-  packages: write
-
-jobs:
-  docker-syncthing:
-    name: Build and push Docker images
-    if: github.repository == 'syncthing/syncthing'
-    runs-on: ubuntu-latest
-    environment: release
-    strategy:
-      matrix:
-        pkg:
-          - stcrashreceiver
-          - strelaypoolsrv
-          - stupgrades
-          - ursrv
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          check-latest: true
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Login to GHCR
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build binaries
-        run: |
-          for arch in arm64 amd64; do
-            go run build.go -goos linux -goarch "$arch" build ${{ matrix.pkg }}
-            mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-"$arch"
-          done
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Set Docker tags (all branches)
-        run: |
-          tags=docker.io/syncthing/${{ matrix.pkg }}:${{ github.sha }},ghcr.io/syncthing/infra/${{ matrix.pkg }}:${{ github.sha }}
-          echo "TAGS=$tags" >> $GITHUB_ENV
-
-      - name: Set Docker tags (latest)
-        if: github.ref == 'refs/heads/infrastructure'
-        run: |
-          tags=docker.io/syncthing/${{ matrix.pkg }}:latest,ghcr.io/syncthing/infra/${{ matrix.pkg }}:latest,${{ env.TAGS }}
-          echo "TAGS=$tags" >> $GITHUB_ENV
-
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./Dockerfile.${{ matrix.pkg }}
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ env.TAGS }}
-          labels: |
-            org.opencontainers.image.revision=${{ github.sha }}

.github/workflows/build-syncthing.yaml

@@ -1,976 +0,0 @@
-name: Build Syncthing
-
-on:
-  pull_request:
-  push:
-  schedule:
-    # Run nightly build at 05:00 UTC
-    - cron: '00 05 * * *'
-  workflow_dispatch:
-
-env:
-  # The go version to use for builds. We set check-latest to true when
-  # installing, so we get the latest patch version that matches the
-  # expression.
-  GO_VERSION: "~1.23.0"
-
-  # Optimize compatibility on the slow archictures.
-  GO386: softfloat
-  GOARM: "5"
-  GOMIPS: softfloat
-
-  # Avoid hilarious amounts of obscuring log output when running tests.
-  LOGGER_DISCARD: "1"
-
-  # Our build metadata
-  BUILD_USER: builder
-  BUILD_HOST: github.syncthing.net
-
-# A note on actions and third party code... The actions under actions/ (like
-# `uses: actions/checkout`) are maintained by GitHub, and we need to trust
-# GitHub to maintain their code and infrastructure or we're in deep shit in
-# general. The same doesn't necessarily apply to other actions authors, so
-# some care needs to be taken when adding steps, especially in the paths
-# that lead up to code being packaged and signed.
-
-jobs:
-
-  #
-  # Tests for all platforms. Runs a matrix build on Windows, Linux and Mac,
-  # with the list of expected supported Go versions (current, previous).
-  #
-
-  build-test:
-    name: Build and test
-    strategy:
-      fail-fast: false
-      matrix:
-        runner: ["windows-latest", "ubuntu-latest", "macos-latest"]
-        # The oldest version in this list should match what we have in our go.mod.
-        # Variables don't seem to be supported here, or we could have done something nice.
-        go: ["~1.22.6", "~1.23.0"]
-    runs-on: ${{ matrix.runner }}
-    steps:
-      - name: Set git to use LF
-        if: matrix.runner == 'windows-latest'
-        # Without this, the Windows checkout will happen with CRLF line
-        # endings, which is fine for the source code but messes up tests
-        # that depend on data on disk being as expected. Ideally, those
-        # tests should be fixed, but not today.
-        run: |
-          git config --global core.autocrlf false
-          git config --global core.eol lf
-
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ matrix.go }}
-          cache: true
-          check-latest: true
-
-      - name: Build
-        run: |
-          go run build.go
-
-      - name: Install go-test-json-to-loki
-        run: |
-          go install calmh.dev/go-test-json-to-loki@latest
-
-      - name: Test
-        run: |
-          go version
-          go run build.go test | go-test-json-to-loki
-        env:
-          GOFLAGS: "-json"
-          LOKI_URL: ${{ secrets.LOKI_URL }}
-          LOKI_USER: ${{ secrets.LOKI_USER }}
-          LOKI_PASSWORD: ${{ secrets.LOKI_PASSWORD }}
-          LOKI_LABELS: "go=${{ matrix.go }},runner=${{ matrix.runner }},repo=${{ github.repository }},ref=${{ github.ref }}"
-
-  #
-  # Meta checks for formatting, copyright, etc
-  #
-
-  correctness:
-    name: Check correctness
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Check correctness
-        run: |
-          go test -v ./meta
-
-  #
-  # The basic checks job is a virtual one that depends on the matrix tests,
-  # the correctness checks, and various builds that we always do. This makes
-  # it easy to have the PR process have a single test as a gatekeeper for
-  # merging, instead of having to add all the matrix tests and update them
-  # each time the version changes. (The top level test is not available for
-  # choosing there, only the matrix "children".)
-  #
-
-  basics:
-    name: Basic checks passed
-    runs-on: ubuntu-latest
-    needs:
-      - build-test
-      - correctness
-      - package-linux
-      - package-cross
-      - package-source
-      - package-debian
-      - govulncheck
-    steps:
-      - uses: actions/checkout@v4
-
-  #
-  # Windows
-  #
-
-  package-windows:
-    name: Package for Windows
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
-    environment: release
-    runs-on: windows-latest
-    steps:
-      - name: Set git to use LF
-        # Without this, the checkout will happen with CRLF line endings,
-        # which is fine for the source code but messes up tests that depend
-        # on data on disk being as expected. Ideally, those tests should be
-        # fixed, but not today.
-        run: |
-          git config --global core.autocrlf false
-          git config --global core.eol lf
-
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ github.ref }} # https://github.com/actions/checkout/issues/882
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Get actual Go version
-        run: |
-          go version
-          echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
-
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ~\AppData\Local\go-build
-            ~\go\pkg\mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
-
-      - name: Install dependencies
-        run: |
-          go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@v1.4.0
-
-      - name: Create packages
-        run: |
-          $targets = 'syncthing', 'stdiscosrv', 'strelaysrv'
-          $archs = 'amd64', 'arm', 'arm64', '386'
-          foreach ($arch in $archs) {
-            foreach ($tgt in $targets) {
-              go run build.go -goarch $arch zip $tgt
-            }
-          }
-        env:
-          CGO_ENABLED: "0"
-          CODESIGN_SIGNTOOL: ${{ secrets.CODESIGN_SIGNTOOL }}
-          CODESIGN_CERTIFICATE_BASE64: ${{ secrets.CODESIGN_CERTIFICATE_BASE64 }}
-          CODESIGN_CERTIFICATE_PASSWORD: ${{ secrets.CODESIGN_CERTIFICATE_PASSWORD }}
-          CODESIGN_TIMESTAMP_SERVER: ${{ secrets.CODESIGN_TIMESTAMP_SERVER }}
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: packages-windows
-          path: "*.zip"
-
-  #
-  # Linux
-  #
-
-  package-linux:
-    name: Package for Linux
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ github.ref }} # https://github.com/actions/checkout/issues/882
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Get actual Go version
-        run: |
-          go version
-          echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
-
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
-
-      - name: Create packages
-        run: |
-          archs=$(go tool dist list | grep linux | sed 's#linux/##')
-          for goarch in $archs ; do
-            for tgt in syncthing stdiscosrv strelaysrv ; do
-              go run build.go -goarch "$goarch" tar "$tgt"
-            done
-          done
-        env:
-          CGO_ENABLED: "0"
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: packages-linux
-          path: |
-            *.tar.gz
-            compat.json
-
-  #
-  # macOS
-  #
-
-  package-macos:
-    name: Package for macOS
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
-    environment: release
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ github.ref }} # https://github.com/actions/checkout/issues/882
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Get actual Go version
-        run: |
-          go version
-          echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
-
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
-
-      - name: Import signing certificate
-        run: |
-          # Set up a run-specific keychain, making it available for the
-          # `codesign` tool.
-          umask 066
-          KEYCHAIN_PATH=$RUNNER_TEMP/codesign.keychain
-          KEYCHAIN_PASSWORD=$(uuidgen)
-          security create-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
-          security default-keychain -s "$KEYCHAIN_PATH"
-          security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
-          security set-keychain-settings -lut 21600 "$KEYCHAIN_PATH"
-
-          # Import the certificate
-          CERTIFICATE_PATH=$RUNNER_TEMP/codesign.p12
-          echo "$DEVELOPER_ID_CERTIFICATE_BASE64" | base64 -d -o "$CERTIFICATE_PATH"
-          security import "$CERTIFICATE_PATH" -k "$KEYCHAIN_PATH" -P "$DEVELOPER_ID_CERTIFICATE_PASSWORD" -T /usr/bin/codesign -T /usr/bin/productsign
-          security set-key-partition-list -S apple-tool:,apple: -s -k actions "$KEYCHAIN_PATH"
-
-          # Set the codesign identity for following steps
-          echo "CODESIGN_IDENTITY=$CODESIGN_IDENTITY" >> $GITHUB_ENV
-        env:
-          DEVELOPER_ID_CERTIFICATE_BASE64: ${{ secrets.DEVELOPER_ID_CERTIFICATE_BASE64 }}
-          DEVELOPER_ID_CERTIFICATE_PASSWORD: ${{ secrets.DEVELOPER_ID_CERTIFICATE_PASSWORD }}
-          CODESIGN_IDENTITY: ${{ secrets.CODESIGN_IDENTITY }}
-
-      - name: Create package (amd64)
-        run: |
-          for tgt in syncthing stdiscosrv strelaysrv ; do
-            go run build.go -goarch amd64 zip "$tgt"
-          done
-        env:
-          CGO_ENABLED: "1"
-
-      - name: Create package (arm64 cross)
-        run: |
-          cat <<EOT > xgo.sh
-          #!/bin/bash
-          CGO_ENABLED=1 \
-            CGO_CFLAGS="-target arm64-apple-macos10.15" \
-            CGO_LDFLAGS="-target arm64-apple-macos10.15" \
-            go "\$@"
-          EOT
-          chmod 755 xgo.sh
-          for tgt in syncthing stdiscosrv strelaysrv ; do
-            go run build.go -gocmd ./xgo.sh -goarch arm64 zip "$tgt"
-          done
-        env:
-          CGO_ENABLED: "1"
-
-      - name: Create package (universal)
-        run: |
-          rm -rf _tmp
-          mkdir _tmp
-          pushd _tmp
-
-          unzip ../syncthing-macos-amd64-*.zip
-          unzip ../syncthing-macos-arm64-*.zip
-          lipo -create syncthing-macos-amd64-*/syncthing syncthing-macos-arm64-*/syncthing -o syncthing
-
-          amd64=(syncthing-macos-amd64-*)
-          universal="${amd64/amd64/universal}"
-          mv "$amd64" "$universal"
-          mv syncthing "$universal"
-          zip -r "../$universal.zip" "$universal"
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: packages-macos
-          path: "*.zip"
-
-  notarize-macos:
-    name: Notarize for macOS
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
-    environment: release
-    needs:
-      - package-macos
-    runs-on: macos-latest
-    steps:
-      - name: Download artifacts
-        uses: actions/download-artifact@v4
-        with:
-          name: packages-macos
-
-      - name: Notarize binaries
-        run: |
-          APPSTORECONNECT_API_KEY_PATH="$RUNNER_TEMP/apikey.p8"
-          echo "$APPSTORECONNECT_API_KEY" | base64 -d -o "$APPSTORECONNECT_API_KEY_PATH"
-          for file in *-macos-*.zip ; do
-            xcrun notarytool submit \
-              -k "$APPSTORECONNECT_API_KEY_PATH" \
-              -d "$APPSTORECONNECT_API_KEY_ID" \
-              -i "$APPSTORECONNECT_API_KEY_ISSUER" \
-              $file
-          done
-        env:
-          APPSTORECONNECT_API_KEY: ${{ secrets.APPSTORECONNECT_API_KEY }}
-          APPSTORECONNECT_API_KEY_ID: ${{ secrets.APPSTORECONNECT_API_KEY_ID }}
-          APPSTORECONNECT_API_KEY_ISSUER: ${{ secrets.APPSTORECONNECT_API_KEY_ISSUER }}
-
-  #
-  # Cross compile other unixes
-  #
-
-  package-cross:
-    name: Package cross compiled
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ github.ref }} # https://github.com/actions/checkout/issues/882
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Get actual Go version
-        run: |
-          go version
-          echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
-
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-cross-${{ hashFiles('**/go.sum') }}
-
-      - name: Create packages
-        run: |
-          platforms=$(go tool dist list \
-            | grep -v aix/ppc64 \
-            | grep -v android/ \
-            | grep -v darwin/ \
-            | grep -v ios/ \
-            | grep -v js/ \
-            | grep -v linux/ \
-            | grep -v nacl/ \
-            | grep -v plan9/ \
-            | grep -v windows/ \
-            | grep -v /wasm \
-          )
-
-          # Build for each platform with errors silenced, because we expect
-          # some oddball platforms to fail. This avoids a bunch of errors in
-          # the GitHub Actions output, instead summarizing each build
-          # failure as a warning.
-          for plat in $platforms; do
-            goos="${plat%/*}"
-            goarch="${plat#*/}"
-            echo "::group ::$plat"
-            for tgt in syncthing stdiscosrv strelaysrv ; do
-              if ! go run build.go -goos "$goos" -goarch "$goarch" tar "$tgt" 2>/dev/null; then
-                echo "::warning ::Failed to build $tgt for $plat"
-              fi
-            done
-            echo "::endgroup::"
-          done
-        env:
-          CGO_ENABLED: "0"
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: packages-other
-          path: "*.tar.gz"
-
-  #
-  # Source
-  #
-
-  package-source:
-    name: Package source code
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ github.ref }} # https://github.com/actions/checkout/issues/882
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Package source
-        run: |
-          version=$(go run build.go version)
-          echo "$version" > RELEASE
-
-          go mod vendor
-          go run build.go assets
-
-          cd ..
-
-          tar c -z -f "syncthing-source-$version.tar.gz" \
-            --exclude .git \
-            syncthing
-
-          mv "syncthing-source-$version.tar.gz" syncthing
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: packages-source
-          path: syncthing-source-*.tar.gz
-
-  #
-  # Sign binaries for auto upgrade, generate ASC signature files
-  #
-
-  sign-for-upgrade:
-    name: Sign for upgrade
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-')  || startsWith(github.ref, 'refs/tags/v'))
-    environment: release
-    needs:
-      - package-windows
-      - package-linux
-      - package-macos
-      - package-cross
-      - package-source
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ github.ref }} # https://github.com/actions/checkout/issues/882
-
-      - uses: actions/checkout@v4
-        with:
-          repository: syncthing/release-tools
-          path: tools
-          fetch-depth: 0
-
-      - name: Download artifacts
-        uses: actions/download-artifact@v4
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Install signing tool
-        run: |
-          go install ./cmd/dev/stsigtool
-
-      - name: Sign archives
-        run: |
-          export PRIVATE_KEY="$RUNNER_TEMP/privkey.pem"
-          export PATH="$PATH:$(go env GOPATH)/bin"
-          echo "$STSIGTOOL_PRIVATE_KEY" | base64 -d > "$PRIVATE_KEY"
-          mkdir packages
-          mv packages-*/* packages
-          pushd packages
-          "$GITHUB_WORKSPACE/tools/sign-only"
-          rm -f "$PRIVATE_KEY"
-        env:
-          STSIGTOOL_PRIVATE_KEY: ${{ secrets.STSIGTOOL_PRIVATE_KEY }}
-
-      - name: Create shasum files
-        run: |
-          pushd packages
-          files=(*.tar.gz *.zip)
-          sha1sum "${files[@]}" > sha1sum.txt
-          sha256sum "${files[@]}" > sha256sum.txt
-          popd
-
-          version=$(go run build.go version)
-          echo "VERSION=$version" >> $GITHUB_ENV
-
-      - name: Sign shasum files
-        uses: docker://ghcr.io/kastelo/ezapt:latest
-        with:
-          args:
-            sign
-            packages/sha1sum.txt packages/sha256sum.txt
-        env:
-          EZAPT_KEYRING_BASE64: ${{ secrets.APT_GPG_KEYRING_BASE64 }}
-
-      - name: Sign source
-        uses: docker://ghcr.io/kastelo/ezapt:latest
-        with:
-          args:
-            sign --detach --ascii
-            packages/syncthing-source-${{ env.VERSION }}.tar.gz
-        env:
-          EZAPT_KEYRING_BASE64: ${{ secrets.APT_GPG_KEYRING_BASE64 }}
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: packages-signed
-          path: |
-            packages/*.tar.gz
-            packages/*.zip
-            packages/*.asc
-            packages/*.json
-
-  #
-  # Debian
-  #
-
-  package-debian:
-    name: Package for Debian
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ github.ref }} # https://github.com/actions/checkout/issues/882
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Get actual Go version
-        run: |
-          go version
-          echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
-
-      - uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: '3.0'
-
-      - name: Install fpm
-        run: |
-          gem install fpm
-
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-debian-${{ hashFiles('**/go.sum') }}
-
-      - name: Package for Debian
-        run: |
-          for arch in amd64 i386 armhf armel arm64 ; do
-            for tgt in syncthing stdiscosrv strelaysrv ; do
-              go run build.go -no-upgrade -installsuffix=no-upgrade -goarch "$arch" deb "$tgt"
-            done
-          done
-        env:
-          BUILD_USER: debian
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: debian-packages
-          path: "*.deb"
-
-  #
-  # Nightlies
-  #
-
-  publish-nightly:
-    name: Publish nightly build
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && startsWith(github.ref, 'refs/heads/release-nightly')
-    environment: release
-    needs:
-      - sign-for-upgrade
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: syncthing/release-tools
-          path: tools
-          fetch-depth: 0
-
-      - name: Download artifacts
-        uses: actions/download-artifact@v4
-        with:
-          name: packages-signed
-          path: packages
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Create release json
-        run: |
-          cd packages
-          "$GITHUB_WORKSPACE/tools/generate-release-json" "$BASE_URL" > nightly.json
-        env:
-          BASE_URL: ${{ secrets.NIGHTLY_BASE_URL }}
-
-      - name: Push artifacts
-        uses: docker://docker.io/rclone/rclone:latest
-        env:
-          RCLONE_CONFIG_OBJSTORE_TYPE: s3
-          RCLONE_CONFIG_OBJSTORE_PROVIDER: ${{ secrets.S3_PROVIDER }}
-          RCLONE_CONFIG_OBJSTORE_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-          RCLONE_CONFIG_OBJSTORE_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
-          RCLONE_CONFIG_OBJSTORE_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
-          RCLONE_CONFIG_OBJSTORE_REGION: ${{ secrets.S3_REGION }}
-          RCLONE_CONFIG_OBJSTORE_ACL: public-read
-        with:
-          args: sync packages objstore:${{ secrets.S3_BUCKET }}/nightly
-
-  #
-  # Push release artifacts to Spaces
-  #
-
-  publish-release-files:
-    name: Publish release files
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v'))
-    environment: release
-    needs:
-      - sign-for-upgrade
-      - package-debian
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ github.ref }} # https://github.com/actions/checkout/issues/882
-
-      - name: Download signed packages
-        uses: actions/download-artifact@v4
-        with:
-          name: packages-signed
-          path: packages
-
-      - name: Download debian packages
-        uses: actions/download-artifact@v4
-        with:
-          name: debian-packages
-          path: packages
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Set version
-        run: |
-          version=$(go run build.go version)
-          echo "VERSION=$version" >> $GITHUB_ENV
-
-      - name: Push to object store (${{ env.VERSION }})
-        uses: docker://docker.io/rclone/rclone:latest
-        env:
-          RCLONE_CONFIG_OBJSTORE_TYPE: s3
-          RCLONE_CONFIG_OBJSTORE_PROVIDER: ${{ secrets.S3_PROVIDER }}
-          RCLONE_CONFIG_OBJSTORE_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-          RCLONE_CONFIG_OBJSTORE_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
-          RCLONE_CONFIG_OBJSTORE_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
-          RCLONE_CONFIG_OBJSTORE_REGION: ${{ secrets.S3_REGION }}
-          RCLONE_CONFIG_OBJSTORE_ACL: public-read
-        with:
-          args: sync packages objstore:${{ secrets.S3_BUCKET }}/release/${{ env.VERSION }}
-
-      - name: Push to object store (latest)
-        uses: docker://docker.io/rclone/rclone:latest
-        env:
-          RCLONE_CONFIG_OBJSTORE_TYPE: s3
-          RCLONE_CONFIG_OBJSTORE_PROVIDER: ${{ secrets.S3_PROVIDER }}
-          RCLONE_CONFIG_OBJSTORE_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-          RCLONE_CONFIG_OBJSTORE_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
-          RCLONE_CONFIG_OBJSTORE_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
-          RCLONE_CONFIG_OBJSTORE_REGION: ${{ secrets.S3_REGION }}
-          RCLONE_CONFIG_OBJSTORE_ACL: public-read
-        with:
-          args: sync objstore:${{ secrets.S3_BUCKET }}/release/${{ env.VERSION }} objstore:${{ secrets.S3_BUCKET }}/release/latest
-
-  #
-  # Push Debian/APT archive
-  #
-
-  publish-apt:
-    name: Publish APT
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
-    environment: release
-    needs:
-      - package-debian
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ github.ref }} # https://github.com/actions/checkout/issues/882
-
-      - name: Download packages
-        uses: actions/download-artifact@v4
-        with:
-          name: debian-packages
-          path: packages
-
-      - name: Set version
-        run: |
-          version=$(go run build.go version)
-          echo "Version: $version"
-          echo "VERSION=$version" >> $GITHUB_ENV
-
-      # Decide whether packages should go to stable, candidate or nightly
-      - name: Prepare packages
-        run: |
-          kind=stable
-          if [[ $VERSION == *-rc.[0-9] ]] ; then
-            kind=candidate
-          elif [[ $VERSION == *-* ]] ; then
-            kind=nightly
-          fi
-          echo "Kind: $kind"
-          mkdir -p packages/syncthing/$kind
-          mv packages/*.deb packages/syncthing/$kind
-
-      - name: Pull archive
-        uses: docker://docker.io/rclone/rclone:latest
-        env:
-          RCLONE_CONFIG_OBJSTORE_TYPE: s3
-          RCLONE_CONFIG_OBJSTORE_PROVIDER: ${{ secrets.S3_PROVIDER }}
-          RCLONE_CONFIG_OBJSTORE_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-          RCLONE_CONFIG_OBJSTORE_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
-          RCLONE_CONFIG_OBJSTORE_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
-          RCLONE_CONFIG_OBJSTORE_REGION: ${{ secrets.S3_REGION }}
-          RCLONE_CONFIG_OBJSTORE_ACL: public-read
-        with:
-          args: sync objstore:syncthing-apt/dists dists
-
-      - name: Update archive
-        uses: docker://ghcr.io/kastelo/ezapt:latest
-        with:
-          args:
-            publish
-            --add packages
-            --dists dists
-        env:
-          EZAPT_KEYRING_BASE64: ${{ secrets.APT_GPG_KEYRING_BASE64 }}
-
-      - name: Push archive
-        uses: docker://docker.io/rclone/rclone:latest
-        env:
-          RCLONE_CONFIG_OBJSTORE_TYPE: s3
-          RCLONE_CONFIG_OBJSTORE_PROVIDER: ${{ secrets.S3_PROVIDER }}
-          RCLONE_CONFIG_OBJSTORE_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
-          RCLONE_CONFIG_OBJSTORE_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
-          RCLONE_CONFIG_OBJSTORE_ENDPOINT: ${{ secrets.S3_ENDPOINT }}
-          RCLONE_CONFIG_OBJSTORE_REGION: ${{ secrets.S3_REGION }}
-          RCLONE_CONFIG_OBJSTORE_ACL: public-read
-        with:
-          args: sync dists -v objstore:syncthing-apt/dists
-
-  #
-  # Build and push to Docker Hub
-  #
-
-  docker-syncthing:
-    name: Build and push Docker images
-    runs-on: ubuntu-latest
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/release' || github.ref == 'refs/heads/infrastructure' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v'))
-    environment: docker
-    permissions:
-      contents: read
-      packages: write
-    strategy:
-      matrix:
-        pkg:
-          - syncthing
-          - strelaysrv
-          - stdiscosrv
-        include:
-          - pkg: syncthing
-            dockerfile: Dockerfile
-            image: syncthing/syncthing
-          - pkg: strelaysrv
-            dockerfile: Dockerfile.strelaysrv
-            image: syncthing/relaysrv
-          - pkg: stdiscosrv
-            dockerfile: Dockerfile.stdiscosrv
-            image: syncthing/discosrv
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ github.ref }} # https://github.com/actions/checkout/issues/882
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Get actual Go version
-        run: |
-          go version
-          echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
-
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-docker-${{ matrix.pkg }}-${{ hashFiles('**/go.sum') }}
-
-      - name: Build binaries
-        run: |
-          for arch in amd64 arm64 arm; do
-            go run build.go -goos linux -goarch "$arch" -no-upgrade build ${{ matrix.pkg }}
-            mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-"$arch"
-          done
-        env:
-          CGO_ENABLED: "0"
-          BUILD_USER: docker
-
-      - name: Check if we will be able to push images
-        run: |
-          if [[ "${{ secrets.DOCKERHUB_TOKEN }}" != "" ]]; then
-            echo "DOCKER_PUSH=true" >> $GITHUB_ENV;
-          fi
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        if: env.DOCKER_PUSH == 'true'
-        with:
-          registry: docker.io
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Login to GHCR
-        uses: docker/login-action@v3
-        if: env.DOCKER_PUSH == 'true'
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Set version tags
-        run: |
-          version=$(go run build.go version)
-          version=${version#v}
-          if [[ $version == @([0-9]|[0-9][0-9]).@([0-9]|[0-9][0-9]).@([0-9]|[0-9][0-9]) ]] ; then
-            echo Release version, pushing to :latest and version tags
-            major=${version%.*.*}
-            minor=${version%.*}
-            tags=docker.io/${{ matrix.image }}:$version,ghcr.io/${{ matrix.image }}:$version,docker.io/${{ matrix.image }}:$major,ghcr.io/${{ matrix.image }}:$major,docker.io/${{ matrix.image }}:$minor,ghcr.io/${{ matrix.image }}:$minor,docker.io/${{ matrix.image }}:latest,ghcr.io/${{ matrix.image }}:latest
-          elif [[ $version == *-rc.@([0-9]|[0-9][0-9]) ]] ; then
-            echo Release candidate, pushing to :rc and version tags
-            tags=docker.io/${{ matrix.image }}:$version,ghcr.io/${{ matrix.image }}:$version,docker.io/${{ matrix.image }}:rc,ghcr.io/${{ matrix.image }}:rc
-          else
-            echo Development version, pushing to :edge
-            tags=docker.io/${{ matrix.image }}:edge,ghcr.io/${{ matrix.image }}:edge
-          fi
-          echo "DOCKER_TAGS=$tags" >> $GITHUB_ENV
-          echo "VERSION=$version" >> $GITHUB_ENV
-
-      - name: Build and push Docker image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ${{ matrix.dockerfile }}
-          platforms: linux/amd64,linux/arm64,linux/arm/7
-          push: ${{ env.DOCKER_PUSH == 'true' }}
-          tags: ${{ env.DOCKER_TAGS }}
-          labels: |
-            org.opencontainers.image.version=${{ env.VERSION }}
-            org.opencontainers.image.revision=${{ github.sha }}
-
-  #
-  # Check for known vulnerabilities in Go dependencies
-  #
-
-  govulncheck:
-    runs-on: ubuntu-latest
-    name: Run govulncheck
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: run govulncheck
-        run: |
-          go run build.go assets
-          go install golang.org/x/vuln/cmd/govulncheck@latest
-          govulncheck ./...

.github/workflows/trigger-nightly.yaml

@@ -1,21 +0,0 @@
-name: Trigger nightly build & release
-on:
-  workflow_dispatch:
-  schedule:
-    # Run nightly build at 01:00 UTC
-    - cron: '00 01 * * *'
-
-jobs:
-
-  trigger-nightly:
-    runs-on: ubuntu-latest
-    name: Push to release-nightly to trigger build
-    steps:
-
-      - uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
-          fetch-depth: 0
-
-      - run: |
-          git push origin main:release-nightly

.github/workflows/update-docs-translations.yaml

@@ -1,28 +0,0 @@
-name: Update translations and documentation
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: '42 3 * * 1'
-
-jobs:
-
-  update_transifex_docs:
-    runs-on: ubuntu-latest
-    name: Update translations and documentation
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
-      - uses: actions/setup-go@v5
-        with:
-          go-version: stable
-      - run: |
-          set -euo pipefail
-          git config --global user.name 'Syncthing Release Automation'
-          git config --global user.email 'release@syncthing.net'
-          bash build.sh translate
-          bash build.sh prerelease
-          git push
-        env:
-          WEBLATE_TOKEN: ${{ secrets.WEBLATE_TOKEN }}

.gitignore

@@ -1,10 +1,11 @@
 /syncthing
 /stdiscosrv
+syncthing.exe
+stdiscosrv.exe
 *.tar.gz
 *.zip
 *.asc
 *.deb
-*.exe
 .jshintrc
 coverage.out
 files/pidx
@@ -18,4 +19,4 @@ deb
 /repos
 /proto/scripts/protoc-gen-gosyncthing
 /gui/next-gen-gui
-/compat.json
+.idea

.golangci.yml

@@ -1,39 +1,26 @@
+linters-settings:
+  maligned:
+    suggest-new: true
+
 linters:
   enable-all: true
   disable:
-    - cyclop
-    - depguard
-    - exhaustive
-    - exhaustruct
-    - funlen
-    - gci
-    - gochecknoglobals
-    - gochecknoinits
-    - gocognit
-    - goconst
-    - gocyclo
-    - godox
-    - gofmt
     - goimports
-    - gomoddirectives
-    - inamedparam
-    - interfacebloat
-    - ireturn
+    - depguard
     - lll
-    - maintidx
-    - nestif
-    - nonamedreturns
-    - paralleltest
-    - protogetter
+    - gochecknoinits
+    - gochecknoglobals
+    - gofmt
     - scopelint
-    - tagalign
-    - tagliatelle
-    - testpackage
-    - varnamelen
+    - gocyclo
+    - funlen
     - wsl
+    - gocognit
+    - godox
 
-issues:
-  exclude-dirs:
-    - internal/gen
-    - cmd/dev
-    - repos
+service:
+  golangci-lint-version: 1.21.x
+  prepare:
+    - rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
+    - GO111MODULE=on go mod vendor
+    - go run build.go assets

.policy.yml

@@ -1,99 +0,0 @@
-# This is the policy-bot configuration for this repository. It controls
-# which approvals are required for any given pull request. The format is
-# described at https://github.com/palantir/policy-bot. The syntax of the
-# policy can be verified by the bot:
-# curl https://pb.syncthing.net/api/validate -X PUT -T .policy.yml
-
-# The policy below is what is required for any pull request.
-policy:
-  approval:
-    - subject is conventional commit
-    - project metadata requires maintainer approval
-    - or:
-      - is approved by a syncthing contributor
-      - is a translation or dependency update by a contributor
-      - is a trivial change by a contributor
-
-  # Additionally, contributors can disapprove of a PR
-  disapproval:
-    requires:
-      teams:
-        - syncthing/contributors
-
-# The rules for the policy are described below.
-
-approval_rules:
-
-  # All commits (PRs before squashing) should have a valid conventional
-  # commit type subject.
-  - name: subject is conventional commit
-    requires:
-      conditions:
-        title:
-          matches:
-            - '^(feat|fix|docs|chore|refactor|build): [a-z].+'
-            - '^(feat|fix|docs|chore|refactor|build)\(\w+(, \w+)*\): [a-z].+'
-
-  # Changes to important project metadata and documentation, including this
-  # policy, require signoff by a maintainer
-  - name: project metadata requires maintainer approval
-    if:
-      changed_files:
-        paths:
-          - ^[^/]+\.md
-          - ^\.policy\.yml
-          - ^\.github/
-          - ^LICENSE
-    requires:
-      count: 1
-      teams:
-        - syncthing/maintainers
-    options:
-      ignore_update_merges: true
-      allow_contributor: true
-
-  # Regular pull requests require approval by an active contributor
-  - name: is approved by a syncthing contributor
-    requires:
-      count: 1
-      teams:
-        - syncthing/contributors
-    options:
-      ignore_update_merges: true
-      allow_contributor: true
-
-  # Changes to some files (translations, dependencies, compatibility) do not
-  # require approval if they were proposed by a contributor and have a
-  # matching commit subject
-  - name: is a translation or dependency update by a contributor
-    if:
-      only_changed_files:
-        paths:
-          - ^gui/default/assets/lang/
-          - ^go\.mod$
-          - ^go\.sum$
-          - ^compat\.yaml$
-      title:
-        matches:
-          - '^chore\(gui\):'
-          - '^build\(deps\):'
-          - '^build\(compat\):'
-      has_author_in:
-        teams:
-          - syncthing/contributors
-
-  # If the change is small and the label "trivial" is added, we accept that
-  # on trust. These PRs can be audited after the fact as appropriate.
-  # Features are not trivial.
-  - name: is a trivial change by a contributor
-    if:
-      modified_lines:
-        total: "< 25"
-      title:
-        not_matches:
-          - '^feat'
-      has_labels:
-        - trivial
-      has_author_in:
-        teams:
-          - syncthing/contributors

.yamlfmt

@@ -1,4 +0,0 @@
-line_ending: lf
-formatter:
-  type: basic
-  retain_line_breaks: true

AUTHORS

@@ -18,48 +18,37 @@ Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.nor
 Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
 Alan Pope <alan@popey.com>
 Alberto Donato <albertodonato@users.noreply.github.com>
-Aleksey Vasenev <margtu-fivt@ya.ru>
 Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
-Alex Ionescu <github@ionescu.sh>
 Alex Lindeman <139387+aelindeman@users.noreply.github.com>
 Alex Xu <alex.hello71@gmail.com>
 Alexander Graf (alex2108) <register-github@alex-graf.de>
-Alexander Seiler <seileralex@gmail.com>
-Alexandre Alves <alexandrealvesdb.contact@gmail.com>
 Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
 Aman Gupta <aman@tmm1.net>
-Anatoli Babenia <anatoli@rainforce.org>
 Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
-Andreas Sommer <andreas.sommer87@googlemail.com>
 andresvia <andres.via@gmail.com>
 Andrew Dunham (andrew-d) <andrew@du.nham.ca>
-Andrew Meyer <andrewm.bpi@gmail.com>
 Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
 Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
 André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
 andyleap <andyleap@gmail.com>
 Anjan Momi <anjan@momi.ca>
-Anthony Goeckner <agoeckner@users.noreply.github.com>
 Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
 Antony Male (canton7) <antony.male@gmail.com>
 Anur <anurnomeru@163.com>
 Aranjedeath <Aranjedeath@users.noreply.github.com>
 Arkadiusz Tymiński <gevleeog@gmail.com>
-Aroun <login@b-vo.fr>
 Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
 Artur Zubilewicz <AkaZecik@users.noreply.github.com>
 Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
 Aurélien Rainone <476650+arl@users.noreply.github.com>
 BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
 Bart De Vries (mogwa1) <devriesb@gmail.com>
-Beat Reichenbach <44111292+beatreichenbach@users.noreply.github.com>
 Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
 Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
 Ben Shepherd (benshep) <bjashepherd@gmail.com>
 Ben Sidhom (bsidhom) <bsidhom@gmail.com>
 Benedikt Heine (bebehei) <bebe@bebehei.de>
 Benedikt Morbach <benedikt.morbach@googlemail.com>
-Benjamin Nater <17193640+bn4t@users.noreply.github.com>
 Benno Fünfstück <benno.fuenfstueck@gmail.com>
 Benny Ng (tpng) <benny.tpng@gmail.com>
 boomsquared <54829195+boomsquared@users.noreply.github.com>
@@ -71,87 +60,62 @@ Brian R. Becker (brbecker) <brbecker@gmail.com>
 bt90 <btom1990@googlemail.com>
 Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
 Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
-Catfriend1 <16361913+Catfriend1@users.noreply.github.com>
 Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
 Cedric Staniewski (xduugu) <cedric@gmx.ca>
 chenrui <rui@meetup.com>
-Chih-Hsuan Yen <yan12125@gmail.com> <1937689+yan12125@users.noreply.github.com>
+Chih-Hsuan Yen <yan12125@gmail.com>
 Choongkyu <choongkyu.kim+gh@gmail.com> <vapidlyrapid+gh@gmail.com>
 Chris Howie (cdhowie) <me@chrishowie.com>
 Chris Joel (cdata) <chris@scriptolo.gy>
 Chris Tonkinson <chris@masterbran.ch>
-Christian Kujau <ckujau@users.noreply.github.com>
 Christian Prescott <me@christianprescott.com>
 chucic <chucic@seznam.cz>
-cjc7373 <niuchangcun@gmail.com>
 Colin Kennedy (moshen) <moshen.colin@gmail.com>
 Cromefire_ <tim.l@nghorst.net> <26320625+cromefire@users.noreply.github.com>
-cui fliter <imcusg@gmail.com>
 Cyprien Devillez <cypx@users.noreply.github.com>
-d-volution <49024624+d-volution@users.noreply.github.com>
 Dale Visser <dale.visser@live.com>
 Dan <benda.daniel@gmail.com>
-Daniel Barczyk <46358936+DanielBarczyk@users.noreply.github.com>
 Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
 Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
 Daniel Martí (mvdan) <mvdan@mvdan.cc>
-Daniel Padrta <64928366+danpadcz@users.noreply.github.com>
 Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
 David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
 deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com>
-DeflateAwning <11021263+DeflateAwning@users.noreply.github.com>
 Denis A. (dva) <denisva@gmail.com>
 Dennis Wilson (snnd) <dw@risu.io>
 dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com> <27856297+dependabot-preview[bot]@users.noreply.github.com>
 dependabot[bot] <dependabot[bot]@users.noreply.github.com> <49699333+dependabot[bot]@users.noreply.github.com>
 derekriemer <derek.riemer@colorado.edu>
-DerRockWolf <50499906+DerRockWolf@users.noreply.github.com>
 desbma <desbma@users.noreply.github.com>
-Devon G. Redekopp <devon@redekopp.com>
-diemade <spamkill@posteo.ch>
-digital <didev@dinid.net>
-Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com>
 Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
 Domenic Horner <domenic@tgxn.net>
 Dominik Heidler (asdil12) <dominik@heidler.eu>
 Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
 Elliot Huffman <thelich2@gmail.com>
 Emil Hessman (ceh) <emil@hessman.se>
-Emil Lundberg <emil@emlun.se>
-Eng Zer Jun <engzerjun@gmail.com>
-entity0xfe <109791748+entity0xfe@users.noreply.github.com> <entity0xfe@my.domain>
 Eric Lesiuta <elesiuta@gmail.com>
-Eric P <eric@kastelo.net>
 Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
-Evan Spensley <94762716+0evan@users.noreply.github.com>
 Evgeny Kuznetsov <evgeny@kuznetsov.md>
 Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
-Felix <53702818+f-eliks@users.noreply.github.com>
 Felix Ableitner (Nutomic) <me@nutomic.com>
 Felix Lampe <mail@flampe.de>
 Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
 Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
 Frank Isemann (fti7) <frank@isemann.name>
-Gahl Saraf <saraf.gahl@gmail.com> <gahl@raftt.io>
+Gahl Saraf <saraf.gahl@gmail.com>
 georgespatton <georgespatton@users.noreply.github.com>
 ghjklw <malo@jaffre.info>
 Gilli Sigurdsson (gillisig) <gilli@vx.is>
 Gleb Sinyavskiy <zhulik.gleb@gmail.com>
 Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
 greatroar <61184462+greatroar@users.noreply.github.com>
-Greg <gco@jazzhaiku.com>
-guangwu <guoguangwu@magic-shield.com>
-gudvinr <gudvinr@gmail.com>
-Gusted <postmaster@gusted.xyz> <williamzijl7@hotmail.com>
 Han Boetes <han@boetes.org>
 HansK-p <42314815+HansK-p@users.noreply.github.com>
 Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
 Heiko Zuerker (Smiley73) <heiko@zuerker.org>
-Hireworks <129852174+hireworksltd@users.noreply.github.com>
 Hugo Locurcio <hugo.locurcio@hugo.pro>
 Iain Barnett <iainspeed@gmail.com>
 Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
-ignacy123 <ignacy.buczek@onet.pl>
 Ikko Ashimine <eltociear@gmail.com>
 Ilya Brin <464157+ilyabrin@users.noreply.github.com>
 Iskander Sharipov (Alex) <quasilyte@gmail.com>
@@ -160,18 +124,13 @@ Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
 Jack Croft <jccroft1@users.noreply.github.com>
 Jacob <jyundt@gmail.com>
 Jake Peterson (acogdev) <jake@acogdev.com>
-Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net> <jborg@coreweave.com>
-James O'Beirne <wild-github@au92.org>
+Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
 James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
 janost <janost@tuta.io>
 Jaroslav Lichtblau <svetlemodry@users.noreply.github.com>
 Jaroslav Malec (dzarda) <dzardacz@gmail.com>
 jaseg <githubaccount@jaseg.net>
-Jaspitta <ste.scarpitta@gmail.com>
-Jauder Ho <jauderho@users.noreply.github.com>
 Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
-Jaya Kumar <jaya.kumar@ict.nl>
-Jeffery To <jeffery.to@gmail.com>
 jelle van der Waa <jelle@vdwaa.nl>
 Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
 Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
@@ -181,20 +140,15 @@ Johan Andersson <j@i19.se>
 Johan Vromans (sciurius) <jvromans@squirrel.nl>
 John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
 Jonas Thelemann <e-mail@jonas-thelemann.de>
-Jonathan <artback@protonmail.com> <jonagn@gmail.com>
+Jonathan <artback@protonmail.com>
 Jonathan Cross <jcross@gmail.com>
 Jonta <359397+Jonta@users.noreply.github.com>
 Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
-jtagcat <git-514635f7@jtag.cat> <git-12dbd862@jtag.cat>
-Julian Lehrhuber <jul13579@users.noreply.github.com>
+jtagcat <git-514635f7@jtag.cat>
 Jörg Thalheim <Mic92@users.noreply.github.com>
 Jędrzej Kula <kula.jedrek@gmail.com>
-K.B.Dharun Krishna <kbdharunkrishna@gmail.com>
 Kalle Laine <pahakalle@protonmail.com>
-Kapil Sareen <kapilsareen584@gmail.com>
 Karol Różycki (krozycki) <rozycki.karol@gmail.com>
-Kebin Liu <lkebin@gmail.com>
-Keith Harrison <keithh@protonmail.com>
 Keith Turner <kturner@apache.org>
 Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
 Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
@@ -203,7 +157,6 @@ Kevin Bushiri (keevBush) <keevbush@gmail.com> <36192217+keevBush@users.noreply.g
 Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
 klemens <ka7@github.com>
 Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
-kylosus <33132401+kylosus@users.noreply.github.com>
 Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
 Lars Lehtonen <lars.lehtonen@gmail.com>
 Laurent Arnoud <laurent@spkdev.net>
@@ -212,11 +165,7 @@ Leo Arias (elopio) <yo@elopio.net>
 Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
 Lode Hoste (Zillode) <zillode@zillode.be>
 Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
-LSmithx2 <42276854+lsmithx2@users.noreply.github.com>
-luchenhan <168071714+luchenhan@users.noreply.github.com>
 Lukas Lihotzki <lukas@lihotzki.de>
-Luke Hamburg <1992842+luckman212@users.noreply.github.com>
-luzpaz <luzpaz@users.noreply.github.com>
 Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
 Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
 Marc Pujol (kilburn) <kilburn@la3.org>
@@ -225,8 +174,6 @@ marco-m <marco.molteni@laposte.net>
 Marcus Legendre <marcus.legendre@gmail.com>
 Mario Majila <mariustshipichik@gmail.com>
 Mark Pulford (mpx) <mark@kyne.com.au>
-Martchus <martchus@gmx.net>
-Martin Polehla <p0l0us@users.noreply.github.com>
 Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
 Mateusz Ż <thedead4fun@live.com>
 Matic Potočnik <hairyfotr@gmail.com>
@@ -234,29 +181,22 @@ Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
 Matt Robenolt <matt@ydekproductions.com>
 Matteo Ruina <matteo.ruina@gmail.com>
 Maurizio Tomasi <ziotom78@gmail.com>
-Max <github@germancoding.com>
 Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
-maxice8 <30738253+maxice8@users.noreply.github.com>
 MaximAL <almaximal@ya.ru>
 Maxime Thirouin <m@moox.io>
-Maximilian <maxi.rostock@outlook.de> <public@complexvector.space>
 mclang <1721600+mclang@users.noreply.github.com>
 Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
 Michael Ploujnikov (plouj) <ploujj@gmail.com>
 Michael Rienstra <mrienstra@gmail.com>
 Michael Tilli (pyfisch) <pyfisch@gmail.com>
 MichaIng <micha@dietpi.com>
-Migelo <miha@filetki.si>
 Mike Boone <mike@boonedocks.net>
 MikeLund <MikeLund@users.noreply.github.com>
 MikolajTwarog <43782609+MikolajTwarog@users.noreply.github.com>
 Mingxuan Lin <gdlmx@users.noreply.github.com>
 mv1005 <49659413+mv1005@users.noreply.github.com>
 Nate Morrison (nrm21) <natemorrison@gmail.com>
-Naveen <172697+naveensrinivasan@users.noreply.github.com>
-nf <nf@wh3rd.net>
 Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
-Nick Busey <NickBusey@users.noreply.github.com>
 Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
 Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
 Nicolas Perraut <n.perraut@gmail.com>
@@ -266,7 +206,6 @@ NinoM4ster <ninom4ster@gmail.com>
 Nitroretro <43112364+Nitroretro@users.noreply.github.com>
 NoLooseEnds <jon.koslung@gmail.com>
 Oliver Freyermuth <o.freyermuth@googlemail.com>
-orangekame3 <miya.org.0309@gmail.com>
 otbutz <tbutz@optitool.de>
 Otiel <Otiel@users.noreply.github.com>
 overkill <22098433+0verk1ll@users.noreply.github.com>
@@ -290,7 +229,6 @@ Piotr Bejda (piobpl) <piotrb10@gmail.com>
 Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
 Quentin Hibon <qh.public@yahoo.com>
 Rahmi Pruitt <rjpruitt16@gmail.com>
-red_led <red-led@users.noreply.github.com>
 Richard Hartmann <RichiH@users.noreply.github.com>
 Robert Carosi (nov1n) <robert@carosi.nl>
 Roberto Santalla <roobre@users.noreply.github.com>
@@ -299,56 +237,37 @@ Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
 Ross Smith II (rasa) <ross@smithii.com>
 rubenbe <github-com-00ff86@vandamme.email>
 Ruslan Yevdokymov <38809160+ruslanye@users.noreply.github.com>
-Ryan Qian <i@bitbili.net>
 Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
 Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
 Scott Klupfel (kluppy) <kluppy@going2blue.com>
-sec65 <106604020+sec65@users.noreply.github.com>
 Sergey Mishin (ralder) <ralder@yandex.ru>
-Sertonix <83883937+Sertonix@users.noreply.github.com>
-Severin von Wnuck-Lipinski <ss7@live.de>
-Shaarad Dalvi <60266155+shaaraddalvi@users.noreply.github.com> <shdalv@microsoft.com>
+Shaarad Dalvi <60266155+shaaraddalvi@users.noreply.github.com>
 Simon Frei (imsodin) <freisim93@gmail.com>
 Simon Mwepu <simonmwepu@gmail.com>
-Simon Pickup <simon@pickupinfinity.com>
 Sly_tom_cat <slytomcat@mail.ru>
-Sonu Kumar Saw <31889738+dev-saw99@users.noreply.github.com>
 Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
 Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
 Steven Eckhoff <steven.eckhoff.opensource@gmail.com>
 Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
-Sven Bachmann <dev@mcbachmann.de>
-Syncthing Automation <automation@syncthing.net>
-Syncthing Release Automation <release@syncthing.net>
 Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
-Terrance <git@terrance.allofti.me>
-Thomas <9749173+uhthomas@users.noreply.github.com>
 Thomas Hipp <thomashipp@gmail.com>
 Tim Abell (timabell) <tim@timwise.co.uk>
 Tim Howes (timhowes) <timhowes@berkeley.edu>
-Tim Nordenfur <tim@gurka.se>
-Tobias Frölich <40638719+tobifroe@users.noreply.github.com>
 Tobias Klauser <tobias.klauser@gmail.com>
 Tobias Nygren (tnn2) <tnn@nygren.pp.se>
 Tobias Tom (tobiastom) <t.tom@succont.de>
 Tom Jakubowski <tom@crystae.net>
 Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com> <twilczynski@naver.com>
 Tommy Thorn <tommy-github-email@thorn.ws>
-Tommy van der Vorst <tommy-github@pixelspark.nl> <tommy@pixelspark.nl>
 Tully Robinson (tojrobinson) <tully@tojr.org>
 Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
 Tyler Kropp <kropptyler@gmail.com>
 Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
-vapatel2 <149737089+vapatel2@users.noreply.github.com>
 Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
 Victor Buinsky (buinsky) <vix_booja@tut.by>
-Vik <63919734+ViktorOn@users.noreply.github.com>
 Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
-villekalliomaki <53118179+villekalliomaki@users.noreply.github.com>
 Vladimir Rusinov <vrusinov@google.com> <vladimir.rusinov@gmail.com>
 wangguoliang <liangcszzu@163.com>
-WangXi <xib1102@icloud.com>
-Will Rouesnel <wrouesnel@wrouesnel.com>
 William A. Kennington III (wkennington) <william@wkennington.com>
 wouter bolsterlee <wouter@bolsterl.ee>
 Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
@@ -357,4 +276,3 @@ Xavier O. (damajor) <damajor@gmail.com>
 xjtdy888 (xjtdy888) <xjtdy888@163.com>
 Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
 佛跳墙 <daoquan@qq.com>
-落心 <luoxin.ttt@gmail.com>

CONTRIBUTING.md

@@ -24,15 +24,10 @@ too much information will never get you yelled at. :)
 ## Contributing Translations
 
 All translations are done via
-[Weblate](https://hosted.weblate.org/projects/syncthing/). If you wish
-to contribute to a translation, just head over there and sign up.
+[Transifex](https://www.transifex.com/projects/p/syncthing/). If you
+wish to contribute to a translation, just head over there and sign up.
 Before every release, the language resources are updated from the
-latest info on Weblate.
-
-Note that the previously used service at
-[Transifex](https://www.transifex.com/projects/p/syncthing/) is being
-retired and we kindly ask you to sign up on Weblate for continued
-involvement.
+latest info on Transifex.
 
 ## Contributing Code
 

Dockerfile

@@ -1,57 +1,29 @@
 ARG GOVERSION=latest
-
-#
-# Maybe build Syncthing. This is a bit ugly as we can't make an entire
-# section of the Dockerfile conditional, so we end up always pulling the
-# golang image as builder. Then we check if the executable we need already
-# exists (pre-built) otherwise we build it.
-#
-
 FROM golang:$GOVERSION AS builder
-ARG BUILD_USER
-ARG BUILD_HOST
-ARG TARGETARCH
 
 WORKDIR /src
 COPY . .
 
 ENV CGO_ENABLED=0
-RUN if [ ! -f syncthing-linux-$TARGETARCH ] ; then \
-    go run build.go -no-upgrade build syncthing ; \
-    mv syncthing syncthing-linux-$TARGETARCH ; \
-  fi
-
-#
-# The rest of the Dockerfile uses the binary from the builder, prebuilt or
-# not.
-#
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f syncthing && go run build.go -no-upgrade build syncthing
 
 FROM alpine
-ARG TARGETARCH
-
-LABEL org.opencontainers.image.authors="The Syncthing Project" \
-      org.opencontainers.image.url="https://syncthing.net" \
-      org.opencontainers.image.documentation="https://docs.syncthing.net" \
-      org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
-      org.opencontainers.image.vendor="The Syncthing Project" \
-      org.opencontainers.image.licenses="MPL-2.0" \
-      org.opencontainers.image.title="Syncthing"
 
 EXPOSE 8384 22000/tcp 22000/udp 21027/udp
 
 VOLUME ["/var/syncthing"]
 
-RUN apk add --no-cache ca-certificates curl libcap su-exec tzdata
+RUN apk add --no-cache ca-certificates su-exec tzdata
 
-COPY --from=builder /src/syncthing-linux-$TARGETARCH /bin/syncthing
+COPY --from=builder /src/syncthing /bin/syncthing
 COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
 
 ENV PUID=1000 PGID=1000 HOME=/var/syncthing
 
 HEALTHCHECK --interval=1m --timeout=10s \
-  CMD curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -o --color=never OK || exit 1
+  CMD nc -z 127.0.0.1 8384 || exit 1
 
 ENV STGUIADDRESS=0.0.0.0:8384
-ENV STHOMEDIR=/var/syncthing/config
-RUN chmod 755 /bin/entrypoint.sh
-ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing"]
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]

Dockerfile.builder

@@ -1,17 +1,9 @@
 ARG GOVERSION=latest
 FROM golang:$GOVERSION
 
-LABEL org.opencontainers.image.authors="The Syncthing Project" \
-      org.opencontainers.image.url="https://syncthing.net" \
-      org.opencontainers.image.documentation="https://docs.syncthing.net" \
-      org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
-      org.opencontainers.image.vendor="The Syncthing Project" \
-      org.opencontainers.image.licenses="MPL-2.0" \
-      org.opencontainers.image.title="Syncthing Builder"
-
 # FPM to build Debian packages
 RUN apt-get update && apt-get install -y --no-install-recommends \
 	locales rubygems ruby-dev build-essential git \
 	&& apt-get clean \
 	&& rm -rf /var/lib/apt/lists/* \
-	&& gem install fpm
+	&& gem install --no-ri --no-rdoc fpm

Dockerfile.buildx

@@ -0,0 +1,19 @@
+FROM alpine
+ARG TARGETARCH
+
+EXPOSE 8384 22000/tcp 22000/udp 21027/udp
+
+VOLUME ["/var/syncthing"]
+
+RUN apk add --no-cache ca-certificates su-exec tzdata
+
+COPY ./syncthing-linux-$TARGETARCH /bin/syncthing
+COPY ./script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/syncthing
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z 127.0.0.1 8384 || exit 1
+
+ENV STGUIADDRESS=0.0.0.0:8384
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]

Dockerfile.stcrashreceiver

@@ -1,16 +0,0 @@
-FROM alpine
-ARG TARGETARCH
-
-LABEL org.opencontainers.image.authors="The Syncthing Project" \
-      org.opencontainers.image.url="https://syncthing.net" \
-      org.opencontainers.image.documentation="https://docs.syncthing.net" \
-      org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
-      org.opencontainers.image.vendor="The Syncthing Project" \
-      org.opencontainers.image.licenses="MPL-2.0" \
-      org.opencontainers.image.title="Syncthing Crash Receiver"
-
-EXPOSE 8080
-
-COPY stcrashreceiver-linux-${TARGETARCH} /bin/stcrashreceiver
-
-ENTRYPOINT [ "/bin/stcrashreceiver" ]

Dockerfile.stdiscosrv

@@ -1,28 +1,15 @@
 ARG GOVERSION=latest
 FROM golang:$GOVERSION AS builder
-ARG BUILD_USER
-ARG BUILD_HOST
-ARG TARGETARCH
 
 WORKDIR /src
 COPY . .
 
 ENV CGO_ENABLED=0
-RUN if [ ! -f stdiscosrv-linux-$TARGETARCH ] ; then \
-    go run build.go -no-upgrade build stdiscosrv ; \
-    mv stdiscosrv stdiscosrv-linux-$TARGETARCH ; \
-  fi
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f stdiscosrv && go run build.go -no-upgrade build stdiscosrv
 
 FROM alpine
-ARG TARGETARCH
-
-LABEL org.opencontainers.image.authors="The Syncthing Project" \
-      org.opencontainers.image.url="https://syncthing.net" \
-      org.opencontainers.image.documentation="https://docs.syncthing.net" \
-      org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
-      org.opencontainers.image.vendor="The Syncthing Project" \
-      org.opencontainers.image.licenses="MPL-2.0" \
-      org.opencontainers.image.title="Syncthing Discovery Server"
 
 EXPOSE 19200 8443
 
@@ -30,7 +17,7 @@ VOLUME ["/var/stdiscosrv"]
 
 RUN apk add --no-cache ca-certificates su-exec
 
-COPY --from=builder /src/stdiscosrv-linux-$TARGETARCH /bin/stdiscosrv
+COPY --from=builder /src/stdiscosrv /bin/stdiscosrv
 COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
 
 ENV PUID=1000 PGID=1000 HOME=/var/stdiscosrv

Dockerfile.strelaypoolsrv

@@ -1,16 +0,0 @@
-FROM alpine
-ARG TARGETARCH
-
-LABEL org.opencontainers.image.authors="The Syncthing Project" \
-      org.opencontainers.image.url="https://syncthing.net" \
-      org.opencontainers.image.documentation="https://docs.syncthing.net" \
-      org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
-      org.opencontainers.image.vendor="The Syncthing Project" \
-      org.opencontainers.image.licenses="MPL-2.0" \
-      org.opencontainers.image.title="Syncthing Relay Pool Server"
-
-EXPOSE 8080
-
-COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv
-
-ENTRYPOINT ["/bin/strelaypoolsrv", "-listen", ":8080"]

Dockerfile.strelaysrv

@@ -1,28 +1,15 @@
 ARG GOVERSION=latest
 FROM golang:$GOVERSION AS builder
-ARG BUILD_USER
-ARG BUILD_HOST
-ARG TARGETARCH
 
 WORKDIR /src
 COPY . .
 
 ENV CGO_ENABLED=0
-RUN if [ ! -f strelaysrv-linux-$TARGETARCH ] ; then \
-    go run build.go -no-upgrade build strelaysrv ; \
-    mv strelaysrv strelaysrv-linux-$TARGETARCH ; \
-  fi
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f strelaysrv && go run build.go -no-upgrade build strelaysrv
 
 FROM alpine
-ARG TARGETARCH
-
-LABEL org.opencontainers.image.authors="The Syncthing Project" \
-      org.opencontainers.image.url="https://syncthing.net" \
-      org.opencontainers.image.documentation="https://docs.syncthing.net" \
-      org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
-      org.opencontainers.image.vendor="The Syncthing Project" \
-      org.opencontainers.image.licenses="MPL-2.0" \
-      org.opencontainers.image.title="Syncthing Relay Server"
 
 EXPOSE 22067 22070
 
@@ -30,7 +17,7 @@ VOLUME ["/var/strelaysrv"]
 
 RUN apk add --no-cache ca-certificates su-exec
 
-COPY --from=builder /src/strelaysrv-linux-$TARGETARCH /bin/strelaysrv
+COPY --from=builder /src/strelaysrv /bin/strelaysrv
 COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
 
 ENV PUID=1000 PGID=1000 HOME=/var/strelaysrv

Dockerfile.stupgrades

@@ -1,16 +0,0 @@
-FROM alpine
-ARG TARGETARCH
-
-LABEL org.opencontainers.image.authors="The Syncthing Project" \
-      org.opencontainers.image.url="https://syncthing.net" \
-      org.opencontainers.image.documentation="https://docs.syncthing.net" \
-      org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
-      org.opencontainers.image.vendor="The Syncthing Project" \
-      org.opencontainers.image.licenses="MPL-2.0" \
-      org.opencontainers.image.title="Syncthing Upgrades"
-
-EXPOSE 8080
-
-COPY stupgrades-linux-${TARGETARCH} /bin/stupgrades
-
-ENTRYPOINT [ "/bin/stupgrades" ]

Dockerfile.ursrv

@@ -1,16 +0,0 @@
-FROM alpine
-ARG TARGETARCH
-
-LABEL org.opencontainers.image.authors="The Syncthing Project" \
-      org.opencontainers.image.url="https://syncthing.net" \
-      org.opencontainers.image.documentation="https://docs.syncthing.net" \
-      org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
-      org.opencontainers.image.vendor="The Syncthing Project" \
-      org.opencontainers.image.licenses="MPL-2.0" \
-      org.opencontainers.image.title="Syncthing Usage Reporting Server"
-
-EXPOSE 8080
-
-COPY ursrv-linux-${TARGETARCH} /bin/ursrv
-
-ENTRYPOINT [ "/bin/ursrv" ]

GOALS.md

@@ -24,17 +24,17 @@ to avoid corrupting the user's files.
 ### 2. Secure Against Attackers
 
 Again, protecting the user's data is paramount. Regardless of our other
-goals, we must never allow the user's data to be susceptible to eavesdropping
+goals we must never allow the user's data to be susceptible to eavesdropping
 or modification by unauthorized parties.
 
 > This should be understood in context. It is not necessarily reasonable to
 > expect Syncthing to be resistant against well equipped state level
-> attackers. We will, however, do our best. Note also that this is different
+> attackers. We will however do our best. Note also that this is different
 > from anonymity which is not, currently, a goal.
 
 ### 3. Easy to Use
 
-Syncthing should be approachable, understandable, and inclusive.
+Syncthing should be approachable, understandable and inclusive.
 
 > Complex concepts and maths form the base of Syncthing's functionality.
 > This should nonetheless be abstracted or hidden to a degree where
@@ -52,18 +52,18 @@ User interaction should be required only when absolutely necessary.
 ### 5. Universally Available
 
 Syncthing should run on every common computer. We are mindful that the
-latest technology is not always available to every individual.
+latest technology is not always available to any given individual.
 
 > Computers include desktops, laptops, servers, virtual machines, small
 > general purpose computers such as Raspberry Pis and, *where possible*,
-> tablets and phones. NAS appliances, toasters, cars, firearms, thermostats,
-> and so on may include computing capabilities but it is not our goal for
+> tablets and phones. NAS appliances, toasters, cars, firearms, thermostats
+> and so on may include computing capabitilies but it is not our goal for
 > Syncthing to run smoothly on these devices.
 
 ### 6. For Individuals
 
 Syncthing is primarily about empowering the individual user with safe,
-secure, and easy to use file synchronization.
+secure and easy to use file synchronization.
 
 > We acknowledge that it's also useful in an enterprise setting and include
 > functionality to support that. If this is in conflict with the

README-Docker.md

@@ -7,30 +7,23 @@ Use the `/var/syncthing` volume to have the synchronized files available on the
 host. You can add more folders and map them as you prefer.
 
 Note that Syncthing runs as UID 1000 and GID 1000 by default. These may be
-altered with the `PUID` and `PGID` environment variables. In addition
+altered with the ``PUID`` and ``PGID`` environment variables. In addition
 the name of the Syncthing instance can be optionally defined by using
-`--hostname=syncthing` parameter.
-
-To grant Syncthing additional capabilities without running as root, use the
-`PCAP` environment variable with the same syntax as that for `setcap(8)`.
-For example, `PCAP=cap_chown,cap_fowner+ep`.
-
-To set a different umask value, use the `UMASK` environment variable. For
-example `UMASK=002`.
+``--hostname=syncthing`` parameter.
 
 ## Example Usage
 
 **Docker cli**
 ```
 $ docker pull syncthing/syncthing
-$ docker run -p 8384:8384 -p 22000:22000/tcp -p 22000:22000/udp -p 21027:21027/udp \
+$ docker run -p 8384:8384 -p 22000:22000/tcp -p 22000:22000/udp \
     -v /wherever/st-sync:/var/syncthing \
     --hostname=my-syncthing \
     syncthing/syncthing:latest
 ```
 
 **Docker compose**
-```yml
+```
 ---
 version: "3"
 services:
@@ -44,28 +37,20 @@ services:
     volumes:
       - /wherever/st-sync:/var/syncthing
     ports:
-      - 8384:8384 # Web UI
-      - 22000:22000/tcp # TCP file transfers
-      - 22000:22000/udp # QUIC file transfers
-      - 21027:21027/udp # Receive local discovery broadcasts
+      - 8384:8384
+      - 22000:22000/tcp
+      - 22000:22000/udp
     restart: unless-stopped
-    healthcheck:
-      test: curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -o --color=never OK || exit 1
-      interval: 1m
-      timeout: 10s
-      retries: 3
 ```
 
 ## Discovery
 
-Note that Docker's default network mode prevents local IP addresses from
-being discovered, as Syncthing is only able to see the internal IP of the
-container on the `172.17.0.0/16` subnet. This will result in poor transfer rates
-if local device addresses are not manually configured.
+Note that local device discovery will not work with the above command,
+resulting in poor local transfer rates if local device addresses are not
+manually configured.
 
-It is therefore advisable to use the [host network mode](https://docs.docker.com/network/host/) instead:
+To allow local discovery, the docker host network can be used instead:
 
-**Docker cli**
 ```
 $ docker pull syncthing/syncthing
 $ docker run --network=host \
@@ -73,29 +58,6 @@ $ docker run --network=host \
     syncthing/syncthing:latest
 ```
 
-**Docker compose**
-```yml
----
-version: "3"
-services:
-  syncthing:
-    image: syncthing/syncthing
-    container_name: syncthing
-    hostname: my-syncthing
-    environment:
-      - PUID=1000
-      - PGID=1000
-    volumes:
-      - /wherever/st-sync:/var/syncthing
-    network_mode: host
-    restart: unless-stopped
-    healthcheck:
-      test: curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -o --color=never OK || exit 1
-      interval: 1m
-      timeout: 10s
-      retries: 3
-```
-
 Be aware that syncthing alone is now in control of what interfaces and ports it
 listens on. You can edit the syncthing configuration to change the defaults if
 there are conflicts.

README.md

@@ -2,6 +2,9 @@
 
 ---
 
+[![Latest Linux & Cross Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildLinuxCross.svg?style=flat-square&label=linux+%26+cross+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildLinuxCross&guest=1)
+[![Latest Windows Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildWindows.svg?style=flat-square&label=windows+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildWindows&guest=1)
+[![Latest Mac Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildMac.svg?style=flat-square&label=mac+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildMac&guest=1)
 [![MPLv2 License](https://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/88/badge)](https://bestpractices.coreinfrastructure.org/projects/88)
 [![Go Report Card](https://goreportcard.com/badge/github.com/syncthing/syncthing)](https://goreportcard.com/report/github.com/syncthing/syncthing)
@@ -10,8 +13,8 @@
 
 Syncthing is a **continuous file synchronization program**. It synchronizes
 files between two or more computers. We strive to fulfill the goals below.
-The goals are listed in order of importance, the most important ones first.
-This is the summary version of the goal list - for more
+The goals are listed in order of importance, the most important one being
+the first. This is the summary version of the goal list - for more
 commentary, see the full [Goals document][13].
 
 Syncthing should be:
@@ -24,12 +27,12 @@ Syncthing should be:
 2. **Secure Against Attackers**
 
    Again, protecting the user's data is paramount. Regardless of our other
-   goals, we must never allow the user's data to be susceptible to
+   goals we must never allow the user's data to be susceptible to
    eavesdropping or modification by unauthorized parties.
 
 3. **Easy to Use**
 
-   Syncthing should be approachable, understandable, and inclusive.
+   Syncthing should be approachable, understandable and inclusive.
 
 4. **Automatic**
 
@@ -38,12 +41,12 @@ Syncthing should be:
 5. **Universally Available**
 
    Syncthing should run on every common computer. We are mindful that the
-   latest technology is not always available to every individual.
+   latest technology is not always available to any given individual.
 
 6. **For Individuals**
 
    Syncthing is primarily about empowering the individual user with safe,
-   secure, and easy to use file synchronization.
+   secure and easy to use file synchronization.
 
 7. **Everything Else**
 
@@ -57,40 +60,41 @@ Take a look at the [getting started guide][2].
 
 There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory][3]. There are also several [GUI
-implementations][11] for Windows, Mac, and Linux.
+implementations][11] for Windows, Mac and Linux.
 
 ## Docker
 
 To run Syncthing in Docker, see [the Docker README][16].
 
+## Vote on features/bugs
+
+We'd like to encourage you to [vote][12] on issues that matter to you.
+This helps the team understand what are the biggest pain points for our users, and could potentially influence what is being worked on next.
+
 ## Getting in Touch
 
 The first and best point of contact is the [Forum][8].
 If you've found something that is clearly a
 bug, feel free to report it in the [GitHub issue tracker][10].
 
-If you believe that you’ve found a Syncthing-related security vulnerability,
-please report it by emailing security@syncthing.net. Do not report it in the
-Forum or issue tracker.
-
 ## Building
 
-Building Syncthing from source is easy. After extracting the source bundle from
-a release or checking out git, you just need to run `go run build.go` and the
-binaries are created in `./bin`. There's [a guide][5] with more details on the
-build process.
+Building Syncthing from source is easy, and there's [a guide][5]
+that describes it for both Unix and Windows systems.
 
 ## Signed Releases
 
-Release binaries are GPG signed with the key available from
-https://syncthing.net/security/. There is also a built-in automatic
-upgrade mechanism (disabled in some distribution channels) which uses a
-compiled in ECDSA signature. macOS and Windows binaries are also
-code-signed.
+As of v0.10.15 and onwards release binaries are GPG signed with the key
+D26E6ED000654A3E, available from https://syncthing.net/security.html and
+most key servers.
+
+There is also a built in automatic upgrade mechanism (disabled in some
+distribution channels) which uses a compiled in ECDSA signature. macOS
+binaries are also properly code signed.
 
 ## Documentation
 
-Please see the Syncthing [documentation site][6] [[source]][17].
+Please see the [Syncthing documentation site][6].
 
 All code is licensed under the [MPLv2 License][7].
 
@@ -103,8 +107,9 @@ All code is licensed under the [MPLv2 License][7].
 [8]: https://forum.syncthing.net/
 [10]: https://github.com/syncthing/syncthing/issues
 [11]: https://docs.syncthing.net/users/contrib.html#gui-wrappers
+[12]: https://www.bountysource.com/teams/syncthing/issues
 [13]: https://github.com/syncthing/syncthing/blob/main/GOALS.md
 [14]: assets/logo-text-128.png
 [15]: https://syncthing.net/
 [16]: https://github.com/syncthing/syncthing/blob/main/README-Docker.md
-[17]: https://github.com/syncthing/docs
+

buf.gen.yaml

@@ -1,12 +0,0 @@
-version: v2
-managed:
-  enabled: true
-  override:
-    - file_option: go_package_prefix
-      value: github.com/syncthing/syncthing/internal/gen
-plugins:
-  - remote: buf.build/protocolbuffers/go:v1.35.1
-    out: .
-    opt: module=github.com/syncthing/syncthing
-inputs:
-  - directory: proto

buf.yaml

@@ -1,10 +0,0 @@
-version: v2
-modules:
-  - path: proto
-    name: github.com/syncthing/syncthing
-lint:
-  use:
-    - STANDARD
-breaking:
-  use:
-    - WIRE_JSON

build.go

@@ -4,8 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//go:build tools
-// +build tools
+// +build ignore
 
 package main
 
@@ -15,12 +14,12 @@ import (
 	"bytes"
 	"compress/flate"
 	"compress/gzip"
-	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -32,10 +31,6 @@ import (
 	"strings"
 	"text/template"
 	"time"
-
-	buildpkg "github.com/syncthing/syncthing/lib/build"
-	"github.com/syncthing/syncthing/lib/upgrade"
-	"sigs.k8s.io/yaml"
 )
 
 var (
@@ -52,7 +47,6 @@ var (
 	cc             string
 	run            string
 	benchRun       string
-	buildOut       string
 	debugBinary    bool
 	coverage       bool
 	long           bool
@@ -86,6 +80,7 @@ var targets = map[string]target{
 	"all": {
 		// Only valid for the "build" and "install" commands as it lacks all
 		// the archive creation stuff. buildPkgs gets filled out in init()
+		tags: []string{"purego"},
 	},
 	"syncthing": {
 		// The default target for "build", "install", "tar", "zip", "deb", etc.
@@ -96,40 +91,41 @@ var targets = map[string]target{
 		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/syncthing"},
 		binaryName:  "syncthing", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
-			{src: "{{binary}}", dst: "{{binary}}", perm: 0o755},
-			{src: "README.md", dst: "README.txt", perm: 0o644},
-			{src: "LICENSE", dst: "LICENSE.txt", perm: 0o644},
-			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0o644},
+			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
+			{src: "README.md", dst: "README.txt", perm: 0644},
+			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 			// All files from etc/ and extra/ added automatically in init().
 		},
 		systemdService: "syncthing@*.service",
 		installationFiles: []archiveFile{
-			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0o755},
-			{src: "README.md", dst: "deb/usr/share/doc/syncthing/README.txt", perm: 0o644},
-			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing/LICENSE.txt", perm: 0o644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing/AUTHORS.txt", perm: 0o644},
-			{src: "man/syncthing.1", dst: "deb/usr/share/man/man1/syncthing.1", perm: 0o644},
-			{src: "man/syncthing-config.5", dst: "deb/usr/share/man/man5/syncthing-config.5", perm: 0o644},
-			{src: "man/syncthing-stignore.5", dst: "deb/usr/share/man/man5/syncthing-stignore.5", perm: 0o644},
-			{src: "man/syncthing-device-ids.7", dst: "deb/usr/share/man/man7/syncthing-device-ids.7", perm: 0o644},
-			{src: "man/syncthing-event-api.7", dst: "deb/usr/share/man/man7/syncthing-event-api.7", perm: 0o644},
-			{src: "man/syncthing-faq.7", dst: "deb/usr/share/man/man7/syncthing-faq.7", perm: 0o644},
-			{src: "man/syncthing-networking.7", dst: "deb/usr/share/man/man7/syncthing-networking.7", perm: 0o644},
-			{src: "man/syncthing-rest-api.7", dst: "deb/usr/share/man/man7/syncthing-rest-api.7", perm: 0o644},
-			{src: "man/syncthing-security.7", dst: "deb/usr/share/man/man7/syncthing-security.7", perm: 0o644},
-			{src: "man/syncthing-versioning.7", dst: "deb/usr/share/man/man7/syncthing-versioning.7", perm: 0o644},
-			{src: "etc/linux-systemd/system/syncthing@.service", dst: "deb/lib/systemd/system/syncthing@.service", perm: 0o644},
-			{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0o644},
-			{src: "etc/linux-sysctl/30-syncthing.conf", dst: "deb/usr/lib/sysctl.d/30-syncthing.conf", perm: 0o644},
-			{src: "etc/firewall-ufw/syncthing", dst: "deb/etc/ufw/applications.d/syncthing", perm: 0o644},
-			{src: "etc/linux-desktop/syncthing-start.desktop", dst: "deb/usr/share/applications/syncthing-start.desktop", perm: 0o644},
-			{src: "etc/linux-desktop/syncthing-ui.desktop", dst: "deb/usr/share/applications/syncthing-ui.desktop", perm: 0o644},
-			{src: "assets/logo-32.png", dst: "deb/usr/share/icons/hicolor/32x32/apps/syncthing.png", perm: 0o644},
-			{src: "assets/logo-64.png", dst: "deb/usr/share/icons/hicolor/64x64/apps/syncthing.png", perm: 0o644},
-			{src: "assets/logo-128.png", dst: "deb/usr/share/icons/hicolor/128x128/apps/syncthing.png", perm: 0o644},
-			{src: "assets/logo-256.png", dst: "deb/usr/share/icons/hicolor/256x256/apps/syncthing.png", perm: 0o644},
-			{src: "assets/logo-512.png", dst: "deb/usr/share/icons/hicolor/512x512/apps/syncthing.png", perm: 0o644},
-			{src: "assets/logo-only.svg", dst: "deb/usr/share/icons/hicolor/scalable/apps/syncthing.svg", perm: 0o644},
+			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
+			{src: "README.md", dst: "deb/usr/share/doc/syncthing/README.txt", perm: 0644},
+			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing/AUTHORS.txt", perm: 0644},
+			{src: "man/syncthing.1", dst: "deb/usr/share/man/man1/syncthing.1", perm: 0644},
+			{src: "man/syncthing-config.5", dst: "deb/usr/share/man/man5/syncthing-config.5", perm: 0644},
+			{src: "man/syncthing-stignore.5", dst: "deb/usr/share/man/man5/syncthing-stignore.5", perm: 0644},
+			{src: "man/syncthing-device-ids.7", dst: "deb/usr/share/man/man7/syncthing-device-ids.7", perm: 0644},
+			{src: "man/syncthing-event-api.7", dst: "deb/usr/share/man/man7/syncthing-event-api.7", perm: 0644},
+			{src: "man/syncthing-faq.7", dst: "deb/usr/share/man/man7/syncthing-faq.7", perm: 0644},
+			{src: "man/syncthing-networking.7", dst: "deb/usr/share/man/man7/syncthing-networking.7", perm: 0644},
+			{src: "man/syncthing-rest-api.7", dst: "deb/usr/share/man/man7/syncthing-rest-api.7", perm: 0644},
+			{src: "man/syncthing-security.7", dst: "deb/usr/share/man/man7/syncthing-security.7", perm: 0644},
+			{src: "man/syncthing-versioning.7", dst: "deb/usr/share/man/man7/syncthing-versioning.7", perm: 0644},
+			{src: "etc/linux-systemd/system/syncthing@.service", dst: "deb/lib/systemd/system/syncthing@.service", perm: 0644},
+			{src: "etc/linux-systemd/system/syncthing-resume.service", dst: "deb/lib/systemd/system/syncthing-resume.service", perm: 0644},
+			{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0644},
+			{src: "etc/linux-sysctl/30-syncthing.conf", dst: "deb/usr/lib/sysctl.d/30-syncthing.conf", perm: 0644},
+			{src: "etc/firewall-ufw/syncthing", dst: "deb/etc/ufw/applications.d/syncthing", perm: 0644},
+			{src: "etc/linux-desktop/syncthing-start.desktop", dst: "deb/usr/share/applications/syncthing-start.desktop", perm: 0644},
+			{src: "etc/linux-desktop/syncthing-ui.desktop", dst: "deb/usr/share/applications/syncthing-ui.desktop", perm: 0644},
+			{src: "assets/logo-32.png", dst: "deb/usr/share/icons/hicolor/32x32/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-64.png", dst: "deb/usr/share/icons/hicolor/64x64/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-128.png", dst: "deb/usr/share/icons/hicolor/128x128/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-256.png", dst: "deb/usr/share/icons/hicolor/256x256/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-512.png", dst: "deb/usr/share/icons/hicolor/512x512/apps/syncthing.png", perm: 0644},
+			{src: "assets/logo-only.svg", dst: "deb/usr/share/icons/hicolor/scalable/apps/syncthing.svg", perm: 0644},
 		},
 	},
 	"stdiscosrv": {
@@ -141,22 +137,23 @@ var targets = map[string]target{
 		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/stdiscosrv"},
 		binaryName:  "stdiscosrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
-			{src: "{{binary}}", dst: "{{binary}}", perm: 0o755},
-			{src: "cmd/stdiscosrv/README.md", dst: "README.txt", perm: 0o644},
-			{src: "LICENSE", dst: "LICENSE.txt", perm: 0o644},
-			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0o644},
+			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
+			{src: "cmd/stdiscosrv/README.md", dst: "README.txt", perm: 0644},
+			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
-		systemdService: "stdiscosrv.service",
+		systemdService: "cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service",
 		installationFiles: []archiveFile{
-			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0o755},
-			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0o644},
-			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0o644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0o644},
-			{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0o644},
-			{src: "cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service", dst: "deb/lib/systemd/system/stdiscosrv.service", perm: 0o644},
-			{src: "cmd/stdiscosrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-discosrv", perm: 0o644},
-			{src: "cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv", dst: "deb/etc/ufw/applications.d/stdiscosrv", perm: 0o644},
+			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
+			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0644},
+			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0644},
+			{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service", dst: "deb/lib/systemd/system/stdiscosrv.service", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-discosrv", perm: 0644},
+			{src: "cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv", dst: "deb/etc/ufw/applications.d/stdiscosrv", perm: 0644},
 		},
+		tags: []string{"purego"},
 	},
 	"strelaysrv": {
 		name:        "strelaysrv",
@@ -167,48 +164,44 @@ var targets = map[string]target{
 		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaysrv"},
 		binaryName:  "strelaysrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
-			{src: "{{binary}}", dst: "{{binary}}", perm: 0o755},
-			{src: "cmd/strelaysrv/README.md", dst: "README.txt", perm: 0o644},
-			{src: "cmd/strelaysrv/LICENSE", dst: "LICENSE.txt", perm: 0o644},
-			{src: "LICENSE", dst: "LICENSE.txt", perm: 0o644},
-			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0o644},
+			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
+			{src: "cmd/strelaysrv/README.md", dst: "README.txt", perm: 0644},
+			{src: "cmd/strelaysrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
-		systemdService: "strelaysrv.service",
+		systemdService: "cmd/strelaysrv/etc/linux-systemd/strelaysrv.service",
 		installationFiles: []archiveFile{
-			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0o755},
-			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0o644},
-			{src: "cmd/strelaysrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0o644},
-			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0o644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0o644},
-			{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0o644},
-			{src: "cmd/strelaysrv/etc/linux-systemd/strelaysrv.service", dst: "deb/lib/systemd/system/strelaysrv.service", perm: 0o644},
-			{src: "cmd/strelaysrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-relaysrv", perm: 0o644},
-			{src: "cmd/strelaysrv/etc/firewall-ufw/strelaysrv", dst: "deb/etc/ufw/applications.d/strelaysrv", perm: 0o644},
+			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
+			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0644},
+			{src: "cmd/strelaysrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
+			{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0644},
+			{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0644},
+			{src: "cmd/strelaysrv/etc/linux-systemd/strelaysrv.service", dst: "deb/lib/systemd/system/strelaysrv.service", perm: 0644},
+			{src: "cmd/strelaysrv/etc/linux-systemd/default", dst: "deb/etc/default/syncthing-relaysrv", perm: 0644},
+			{src: "cmd/strelaysrv/etc/firewall-ufw/strelaysrv", dst: "deb/etc/ufw/applications.d/strelaysrv", perm: 0644},
 		},
 	},
 	"strelaypoolsrv": {
 		name:        "strelaypoolsrv",
+		debname:     "syncthing-relaypoolsrv",
+		debdeps:     []string{"libc6"},
 		description: "Syncthing Relay Pool Server",
-		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/infra/strelaypoolsrv"},
-		binaryName:  "strelaypoolsrv",
-	},
-	"stupgrades": {
-		name:        "stupgrades",
-		description: "Syncthing Upgrade Check Server",
-		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/infra/stupgrades"},
-		binaryName:  "stupgrades",
-	},
-	"stcrashreceiver": {
-		name:        "stcrashreceiver",
-		description: "Syncthing Crash Server",
-		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/infra/stcrashreceiver"},
-		binaryName:  "stcrashreceiver",
-	},
-	"ursrv": {
-		name:        "ursrv",
-		description: "Syncthing Usage Reporting Server",
-		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/infra/ursrv"},
-		binaryName:  "ursrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaypoolsrv"},
+		binaryName:  "strelaypoolsrv", // .exe will be added automatically for Windows builds
+		archiveFiles: []archiveFile{
+			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
+			{src: "cmd/strelaypoolsrv/README.md", dst: "README.txt", perm: 0644},
+			{src: "cmd/strelaypoolsrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
+		},
+		installationFiles: []archiveFile{
+			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
+			{src: "cmd/strelaypoolsrv/README.md", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/README.txt", perm: 0644},
+			{src: "cmd/strelaypoolsrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/AUTHORS.txt", perm: 0644},
+		},
 	},
 }
 
@@ -216,11 +209,15 @@ func initTargets() {
 	all := targets["all"]
 	pkgs, _ := filepath.Glob("cmd/*")
 	for _, pkg := range pkgs {
-		if files, err := filepath.Glob(pkg + "/*.go"); err != nil || len(files) == 0 {
-			// No go files in the directory
+		pkg = filepath.Base(pkg)
+		if strings.HasPrefix(pkg, ".") {
+			// ignore dotfiles
 			continue
 		}
-		all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/%s", pkg))
+		if noupgrade && pkg == "stupgrades" {
+			continue
+		}
+		all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/cmd/%s", pkg))
 	}
 	targets["all"] = all
 
@@ -228,13 +225,13 @@ func initTargets() {
 	// and "extra" dirs.
 	syncthingPkg := targets["syncthing"]
 	for _, file := range listFiles("etc") {
-		syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0o644})
+		syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0644})
 	}
 	for _, file := range listFiles("extra") {
-		syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0o644})
+		syncthingPkg.archiveFiles = append(syncthingPkg.archiveFiles, archiveFile{src: file, dst: file, perm: 0644})
 	}
 	for _, file := range listFiles("extra") {
-		syncthingPkg.installationFiles = append(syncthingPkg.installationFiles, archiveFile{src: file, dst: "deb/usr/share/doc/syncthing/" + filepath.Base(file), perm: 0o644})
+		syncthingPkg.installationFiles = append(syncthingPkg.installationFiles, archiveFile{src: file, dst: "deb/usr/share/doc/syncthing/" + filepath.Base(file), perm: 0644})
 	}
 	targets["syncthing"] = syncthingPkg
 }
@@ -304,9 +301,6 @@ func runCommand(cmd string, target target) {
 	case "assets":
 		rebuildAssets()
 
-	case "update-deps":
-		updateDependencies()
-
 	case "proto":
 		proto()
 
@@ -319,16 +313,11 @@ func runCommand(cmd string, target target) {
 	case "transifex":
 		transifex()
 
-	case "weblate":
-		weblate()
-
 	case "tar":
 		buildTar(target, tags)
-		writeCompatJSON()
 
 	case "zip":
 		buildZip(target, tags)
-		writeCompatJSON()
 
 	case "deb":
 		buildDeb(target)
@@ -382,13 +371,13 @@ func parseFlags() {
 	flag.StringVar(&run, "run", "", "Specify which tests to run")
 	flag.StringVar(&benchRun, "bench", "", "Specify which benchmarks to run")
 	flag.BoolVar(&withNextGenGUI, "with-next-gen-gui", withNextGenGUI, "Also build 'newgui'")
-	flag.StringVar(&buildOut, "build-out", "", "Set the '-o' value for 'go build'")
 	flag.Parse()
 }
 
 func test(tags []string, pkgs ...string) {
 	lazyRebuildAssets()
 
+	tags = append(tags, "purego")
 	args := []string{"test", "-tags", strings.Join(tags, " ")}
 	if long {
 		timeout = longTimeout
@@ -399,7 +388,7 @@ func test(tags []string, pkgs ...string) {
 
 	if runtime.GOARCH == "amd64" {
 		switch runtime.GOOS {
-		case buildpkg.Darwin, buildpkg.Linux, buildpkg.FreeBSD: // , "windows": # See https://github.com/golang/go/issues/27089
+		case "darwin", "linux", "freebsd": // , "windows": # See https://github.com/golang/go/issues/27089
 			args = append(args, "-race")
 		}
 	}
@@ -422,7 +411,7 @@ func bench(tags []string, pkgs ...string) {
 func integration(bench bool) {
 	lazyRebuildAssets()
 	args := []string{"test", "-v", "-timeout", "60m", "-tags"}
-	tags := "integration"
+	tags := "purego,integration"
 	if bench {
 		tags += ",benchmark"
 	}
@@ -514,9 +503,6 @@ func build(target target, tags []string) {
 	}
 
 	args := []string{"build", "-v"}
-	if buildOut != "" {
-		args = append(args, "-o", buildOut)
-	}
 	args = appendParameters(args, tags, target.buildPkgs...)
 	runPrint(goCmd, args...)
 }
@@ -733,7 +719,7 @@ func shouldBuildSyso(dir string) (string, error) {
 	}
 
 	jsonPath := filepath.Join(dir, "versioninfo.json")
-	err = os.WriteFile(jsonPath, bs, 0o644)
+	err = ioutil.WriteFile(jsonPath, bs, 0644)
 	if err != nil {
 		return "", errors.New("failed to create " + jsonPath + ": " + err.Error())
 	}
@@ -746,13 +732,7 @@ func shouldBuildSyso(dir string) (string, error) {
 
 	sysoPath := filepath.Join(dir, "cmd", "syncthing", "resource.syso")
 
-	// See https://github.com/josephspurrier/goversioninfo#command-line-flags
-	armOption := ""
-	if strings.Contains(goarch, "arm") {
-		armOption = "-arm=true"
-	}
-
-	if _, err := runError("goversioninfo", "-o", sysoPath, armOption); err != nil {
+	if _, err := runError("goversioninfo", "-o", sysoPath); err != nil {
 		return "", errors.New("failed to create " + sysoPath + ": " + err.Error())
 	}
 
@@ -772,12 +752,12 @@ func shouldCleanupSyso(sysoFilePath string) {
 // exists. The permission bits are copied as well. If dst already exists and
 // the contents are identical to src the modification time is not updated.
 func copyFile(src, dst string, perm os.FileMode) error {
-	in, err := os.ReadFile(src)
+	in, err := ioutil.ReadFile(src)
 	if err != nil {
 		return err
 	}
 
-	out, err := os.ReadFile(dst)
+	out, err := ioutil.ReadFile(dst)
 	if err != nil {
 		// The destination probably doesn't exist, we should create
 		// it.
@@ -792,8 +772,8 @@ func copyFile(src, dst string, perm os.FileMode) error {
 	}
 
 copy:
-	os.MkdirAll(filepath.Dir(dst), 0o777)
-	if err := os.WriteFile(dst, in, perm); err != nil {
+	os.MkdirAll(filepath.Dir(dst), 0777)
+	if err := ioutil.WriteFile(dst, in, perm); err != nil {
 		return err
 	}
 
@@ -817,12 +797,12 @@ func listFiles(dir string) []string {
 
 func rebuildAssets() {
 	os.Setenv("SOURCE_DATE_EPOCH", fmt.Sprint(buildStamp()))
-	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/api/auto", "github.com/syncthing/syncthing/cmd/infra/strelaypoolsrv/auto")
+	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/api/auto", "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto")
 }
 
 func lazyRebuildAssets() {
 	shouldRebuild := shouldRebuildAssets("lib/api/auto/gui.files.go", "gui") ||
-		shouldRebuildAssets("cmd/infra/strelaypoolsrv/auto/gui.files.go", "cmd/infra/strelaypoolsrv/gui")
+		shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.files.go", "cmd/strelaypoolsrv/gui")
 
 	if withNextGenGUI {
 		shouldRebuild = buildNextGenGUI() || shouldRebuild
@@ -852,7 +832,7 @@ func buildNextGenGUI() bool {
 	for _, src := range listFiles("next-gen-gui/dist") {
 		rel, _ := filepath.Rel("next-gen-gui/dist", src)
 		dst := filepath.Join("gui", rel)
-		if err := copyFile(src, dst, 0o644); err != nil {
+		if err := copyFile(src, dst, 0644); err != nil {
 			fmt.Println("copy:", err)
 			os.Exit(1)
 		}
@@ -887,30 +867,23 @@ func shouldRebuildAssets(target, srcdir string) bool {
 	return assetsAreNewer
 }
 
-func updateDependencies() {
-	// Figure out desired Go version
-	bs, err := os.ReadFile("go.mod")
-	if err != nil {
-		log.Fatal(err)
-	}
-	re := regexp.MustCompile(`(?m)^go\s+([0-9.]+)`)
-	matches := re.FindSubmatch(bs)
-	if len(matches) != 2 {
-		log.Fatal("failed to parse go.mod")
-	}
-	goVersion := string(matches[1])
-
-	runPrint(goCmd, "get", "-u", "./...")
-	runPrint(goCmd, "mod", "tidy", "-go="+goVersion, "-compat="+goVersion)
-
-	// We might have updated the protobuf package and should regenerate to match.
-	proto()
-}
-
 func proto() {
-	// buf needs to be installed
-	// https://buf.build/docs/installation/
-	runPrint("buf", "generate")
+	pv := protobufVersion()
+	repo := "https://github.com/gogo/protobuf.git"
+	path := filepath.Join("repos", "protobuf")
+
+	runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
+	os.MkdirAll("repos", 0755)
+
+	if _, err := os.Stat(path); err != nil {
+		runPrint("git", "clone", repo, path)
+	} else {
+		runPrintInDir(path, "git", "fetch")
+	}
+	runPrintInDir(path, "git", "checkout", pv)
+
+	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/cmd/stdiscosrv")
+	runPrint(goCmd, "generate", "proto/generate.go")
 }
 
 func testmocks() {
@@ -943,11 +916,6 @@ func transifex() {
 	runPrint(goCmd, "run", "../../../../script/transifexdl.go")
 }
 
-func weblate() {
-	os.Chdir("gui/default/assets/lang")
-	runPrint(goCmd, "run", "../../../../script/weblatedl.go")
-}
-
 func ldflags(tags []string) string {
 	b := new(strings.Builder)
 	b.WriteString("-w")
@@ -972,7 +940,7 @@ func rmr(paths ...string) {
 }
 
 func getReleaseVersion() (string, error) {
-	bs, err := os.ReadFile("RELEASE")
+	bs, err := ioutil.ReadFile("RELEASE")
 	if err != nil {
 		return "", err
 	}
@@ -995,7 +963,7 @@ func getGitVersion() (string, error) {
 	v0 := string(bs)
 
 	// To be more semantic-versionish and ensure proper ordering in our
-	// upgrade process, we make sure there's only one hyphen in the version.
+	// upgrade process, we make sure there's only one hypen in the version.
 
 	versionRe := regexp.MustCompile(`-([0-9]{1,3}-g[0-9a-f]{5,10}(-dirty)?)`)
 	if m := versionRe.FindStringSubmatch(vcur); len(m) > 0 {
@@ -1083,14 +1051,10 @@ func getBranchSuffix() string {
 
 	branch = parts[len(parts)-1]
 	switch branch {
-	case "release", "main":
+	case "master", "release", "main":
 		// these are not special
 		return ""
 	}
-	if strings.HasPrefix(branch, "release-") {
-		// release branches are not special
-		return ""
-	}
 
 	validBranchRe := regexp.MustCompile(`^[a-zA-Z0-9_.-]+$`)
 	if !validBranchRe.MatchString(branch) {
@@ -1308,11 +1272,11 @@ func zipFile(out string, files []archiveFile) {
 
 		if strings.HasSuffix(f.dst, ".txt") {
 			// Text file. Read it and convert line endings.
-			bs, err := io.ReadAll(sf)
+			bs, err := ioutil.ReadAll(sf)
 			if err != nil {
 				log.Fatal(err)
 			}
-			bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\r', '\n'}, -1)
+			bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\n', '\r'}, -1)
 			fh.UncompressedSize = uint32(len(bs))
 			fh.UncompressedSize64 = uint64(len(bs))
 
@@ -1383,33 +1347,6 @@ func windowsCodesign(file string) {
 		args := []string{"sign", "/fd", algo}
 		if f := os.Getenv("CODESIGN_CERTIFICATE_FILE"); f != "" {
 			args = append(args, "/f", f)
-		} else if b := os.Getenv("CODESIGN_CERTIFICATE_BASE64"); b != "" {
-			// Decode the PFX certificate from base64.
-			bs, err := base64.RawStdEncoding.DecodeString(b)
-			if err != nil {
-				log.Println("Codesign: signing failed: decoding base64:", err)
-				return
-			}
-
-			// Write it to a temporary file
-			f, err := os.CreateTemp("", "codesign-*.pfx")
-			if err != nil {
-				log.Println("Codesign: signing failed: creating temp file:", err)
-				return
-			}
-			_ = f.Chmod(0o600) // best effort remove other users' access
-			defer os.Remove(f.Name())
-			if _, err := f.Write(bs); err != nil {
-				log.Println("Codesign: signing failed: writing temp file:", err)
-				return
-			}
-			if err := f.Close(); err != nil {
-				log.Println("Codesign: signing failed: closing temp file:", err)
-				return
-			}
-
-			// Use that when signing
-			args = append(args, "/f", f.Name())
 		}
 		if p := os.Getenv("CODESIGN_CERTIFICATE_PASSWORD"); p != "" {
 			args = append(args, "/p", p)
@@ -1429,7 +1366,7 @@ func windowsCodesign(file string) {
 
 		bs, err := runError(st, args...)
 		if err != nil {
-			log.Printf("Codesign: signing failed: %v: %s", err, string(bs))
+			log.Println("Codesign: signing failed:", string(bs))
 			return
 		}
 		log.Println("Codesign: successfully signed", file, "using", algo)
@@ -1453,6 +1390,14 @@ func (t target) BinaryName() string {
 	return t.binaryName
 }
 
+func protobufVersion() string {
+	bs, err := runError(goCmd, "list", "-f", "{{.Version}}", "-m", "github.com/gogo/protobuf")
+	if err != nil {
+		log.Fatal("Getting protobuf version:", err)
+	}
+	return string(bs)
+}
+
 func currentAndLatestVersions(n int) ([]string, error) {
 	bs, err := runError("git", "tag", "--sort", "taggerdate")
 	if err != nil {
@@ -1519,29 +1464,3 @@ func nextPatchVersion(ver string) string {
 	digits[len(digits)-1] = strconv.Itoa(n + 1)
 	return strings.Join(digits, ".")
 }
-
-func writeCompatJSON() {
-	bs, err := os.ReadFile("compat.yaml")
-	if err != nil {
-		log.Fatal("Reading compat.yaml:", err)
-	}
-
-	var entries []upgrade.ReleaseCompatibility
-	if err := yaml.Unmarshal(bs, &entries); err != nil {
-		log.Fatal("Parsing compat.yaml:", err)
-	}
-
-	rt := runtime.Version()
-	for _, e := range entries {
-		if !strings.HasPrefix(rt, e.Runtime) {
-			continue
-		}
-		bs, _ := json.MarshalIndent(e, "", "  ")
-		if err := os.WriteFile("compat.json", bs, 0o644); err != nil {
-			log.Fatal("Writing compat.json:", err)
-		}
-		return
-	}
-
-	log.Fatalf("runtime %v not found in compat.yaml", rt)
-}

build.sh

@@ -23,10 +23,10 @@ case "${1:-default}" in
 
 	prerelease)
 		script authors
-		build weblate
+		build transifex
 		pushd man ; ./refresh.sh ; popd
 		git add -A gui man AUTHORS
-		git commit -m 'chore(gui, man, authors): update docs, translations, and contributors'
+		git commit -m 'gui, man, authors: Update docs, translations, and contributors'
 		;;
 
 	*)

cmd/infra/stcrashreceiver/diskstore.go

@@ -1,198 +0,0 @@
-// Copyright (C) 2023 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"io"
-	"log"
-	"math"
-	"os"
-	"path/filepath"
-	"sort"
-	"time"
-)
-
-type diskStore struct {
-	dir      string
-	inbox    chan diskEntry
-	maxBytes int64
-	maxFiles int
-
-	currentFiles []currentFile
-	currentSize  int64
-}
-
-type diskEntry struct {
-	path string
-	data []byte
-}
-
-type currentFile struct {
-	path  string
-	size  int64
-	mtime int64
-}
-
-func (d *diskStore) Serve(ctx context.Context) {
-	if err := os.MkdirAll(d.dir, 0o700); err != nil {
-		log.Println("Creating directory:", err)
-		return
-	}
-
-	if err := d.inventory(); err != nil {
-		log.Println("Failed to inventory disk store:", err)
-	}
-	d.clean()
-
-	cleanTimer := time.NewTicker(time.Minute)
-	inventoryTimer := time.NewTicker(24 * time.Hour)
-
-	buf := new(bytes.Buffer)
-	gw := gzip.NewWriter(buf)
-	for {
-		select {
-		case entry := <-d.inbox:
-			path := d.fullPath(entry.path)
-
-			if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
-				log.Println("Creating directory:", err)
-				continue
-			}
-
-			buf.Reset()
-			gw.Reset(buf)
-			if _, err := gw.Write(entry.data); err != nil {
-				log.Println("Failed to compress crash report:", err)
-				continue
-			}
-			if err := gw.Close(); err != nil {
-				log.Println("Failed to compress crash report:", err)
-				continue
-			}
-			if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
-				log.Printf("Failed to write %s: %v", entry.path, err)
-				_ = os.Remove(path)
-				continue
-			}
-
-			d.currentSize += int64(buf.Len())
-			d.currentFiles = append(d.currentFiles, currentFile{
-				size: int64(len(entry.data)),
-				path: path,
-			})
-
-		case <-cleanTimer.C:
-			d.clean()
-
-		case <-inventoryTimer.C:
-			if err := d.inventory(); err != nil {
-				log.Println("Failed to inventory disk store:", err)
-			}
-
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-func (d *diskStore) Put(path string, data []byte) bool {
-	select {
-	case d.inbox <- diskEntry{
-		path: path,
-		data: data,
-	}:
-		return true
-	default:
-		return false
-	}
-}
-
-func (d *diskStore) Get(path string) ([]byte, error) {
-	path = d.fullPath(path)
-	bs, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	gr, err := gzip.NewReader(bytes.NewReader(bs))
-	if err != nil {
-		return nil, err
-	}
-	defer gr.Close()
-	return io.ReadAll(gr)
-}
-
-func (d *diskStore) Exists(path string) bool {
-	path = d.fullPath(path)
-	_, err := os.Lstat(path)
-	return err == nil
-}
-
-func (d *diskStore) clean() {
-	for len(d.currentFiles) > 0 && (len(d.currentFiles) > d.maxFiles || d.currentSize > d.maxBytes) {
-		f := d.currentFiles[0]
-		log.Println("Removing", f.path)
-		if err := os.Remove(f.path); err != nil {
-			log.Println("Failed to remove file:", err)
-		}
-		d.currentFiles = d.currentFiles[1:]
-		d.currentSize -= f.size
-	}
-	var oldest time.Duration
-	if len(d.currentFiles) > 0 {
-		oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
-	}
-
-	metricDiskstoreFilesTotal.Set(float64(len(d.currentFiles)))
-	metricDiskstoreBytesTotal.Set(float64(d.currentSize))
-	metricDiskstoreOldestAgeSeconds.Set(math.Round(oldest.Seconds()))
-
-	log.Printf("Clean complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
-}
-
-func (d *diskStore) inventory() error {
-	d.currentFiles = nil
-	d.currentSize = 0
-	err := filepath.Walk(d.dir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if info.IsDir() {
-			return nil
-		}
-		if filepath.Ext(path) != ".gz" {
-			return nil
-		}
-		d.currentSize += info.Size()
-		d.currentFiles = append(d.currentFiles, currentFile{
-			path:  path,
-			size:  info.Size(),
-			mtime: info.ModTime().Unix(),
-		})
-		return nil
-	})
-	sort.Slice(d.currentFiles, func(i, j int) bool {
-		return d.currentFiles[i].mtime < d.currentFiles[j].mtime
-	})
-	var oldest time.Duration
-	if len(d.currentFiles) > 0 {
-		oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
-	}
-
-	metricDiskstoreFilesTotal.Set(float64(len(d.currentFiles)))
-	metricDiskstoreBytesTotal.Set(float64(d.currentSize))
-	metricDiskstoreOldestAgeSeconds.Set(math.Round(oldest.Seconds()))
-
-	log.Printf("Inventory complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
-	return err
-}
-
-func (d *diskStore) fullPath(path string) string {
-	return filepath.Join(d.dir, path[0:2], path[2:]) + ".gz"
-}

cmd/infra/stcrashreceiver/main.go

@@ -1,228 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// Command stcrashreceiver is a trivial HTTP server that allows two things:
-//
-// - uploading files (crash reports) named like a SHA256 hash using a PUT request
-// - checking whether such file exists using a HEAD request
-//
-// Typically this should be deployed behind something that manages HTTPS.
-package main
-
-import (
-	"context"
-	"crypto/sha256"
-	"encoding/json"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-	"regexp"
-	"strings"
-
-	"github.com/alecthomas/kong"
-	raven "github.com/getsentry/raven-go"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
-	"github.com/syncthing/syncthing/lib/build"
-	"github.com/syncthing/syncthing/lib/ur"
-)
-
-const maxRequestSize = 1 << 20 // 1 MiB
-
-type cli struct {
-	Dir            string `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."`
-	DSN            string `help:"Sentry DSN" env:"SENTRY_DSN"`
-	Listen         string `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"`
-	MaxDiskFiles   int    `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"`
-	MaxDiskSizeMB  int64  `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"`
-	SentryQueue    int    `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"`
-	DiskQueue      int    `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"`
-	MetricsListen  string `help:"HTTP listen address for metrics" default:":8081" env:"METRICS_LISTEN_ADDRESS"`
-	IngorePatterns string `help:"File containing ignore patterns (regexp)" env:"IGNORE_PATTERNS" type:"existingfile"`
-}
-
-func main() {
-	var params cli
-	kong.Parse(&params)
-
-	mux := http.NewServeMux()
-
-	ds := &diskStore{
-		dir:      filepath.Join(params.Dir, "crash_reports"),
-		inbox:    make(chan diskEntry, params.DiskQueue),
-		maxFiles: params.MaxDiskFiles,
-		maxBytes: params.MaxDiskSizeMB << 20,
-	}
-	go ds.Serve(context.Background())
-
-	ss := &sentryService{
-		dsn:   params.DSN,
-		inbox: make(chan sentryRequest, params.SentryQueue),
-	}
-	go ss.Serve(context.Background())
-
-	var ip *ignorePatterns
-	if params.IngorePatterns != "" {
-		var err error
-		ip, err = loadIgnorePatterns(params.IngorePatterns)
-		if err != nil {
-			log.Fatalf("Failed to load ignore patterns: %v", err)
-		}
-	}
-
-	cr := &crashReceiver{
-		store:  ds,
-		sentry: ss,
-		ignore: ip,
-	}
-
-	mux.Handle("/", cr)
-	mux.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) {
-		w.Write([]byte("OK"))
-	})
-
-	if params.MetricsListen != "" {
-		mmux := http.NewServeMux()
-		mmux.Handle("/metrics", promhttp.Handler())
-		go func() {
-			if err := http.ListenAndServe(params.MetricsListen, mmux); err != nil {
-				log.Fatalln("HTTP serve metrics:", err)
-			}
-		}()
-	}
-
-	if params.DSN != "" {
-		mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports"), ip))
-	}
-
-	log.SetOutput(os.Stdout)
-	if err := http.ListenAndServe(params.Listen, mux); err != nil {
-		log.Fatalln("HTTP serve:", err)
-	}
-}
-
-func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http.ResponseWriter, req *http.Request) {
-	return func(w http.ResponseWriter, req *http.Request) {
-		result := "failure"
-		defer func() {
-			metricFailureReportsTotal.WithLabelValues(result).Inc()
-		}()
-
-		lr := io.LimitReader(req.Body, maxRequestSize)
-		bs, err := io.ReadAll(lr)
-		req.Body.Close()
-		if err != nil {
-			http.Error(w, err.Error(), 500)
-			return
-		}
-
-		if ignore.match(bs) {
-			result = "ignored"
-			return
-		}
-
-		var reports []ur.FailureReport
-		err = json.Unmarshal(bs, &reports)
-		if err != nil {
-			http.Error(w, err.Error(), 400)
-			return
-		}
-		if len(reports) == 0 {
-			// Shouldn't happen
-			log.Printf("Got zero failure reports")
-			return
-		}
-
-		version, err := build.ParseVersion(reports[0].Version)
-		if err != nil {
-			http.Error(w, err.Error(), 400)
-			return
-		}
-		for _, r := range reports {
-			pkt := packet(version, "failure")
-			pkt.Message = r.Description
-			pkt.Extra = raven.Extra{
-				"count": r.Count,
-			}
-			for k, v := range r.Extra {
-				pkt.Extra[k] = v
-			}
-			if r.Goroutines != "" {
-				url, err := saveFailureWithGoroutines(r.FailureData, failureDir)
-				if err != nil {
-					log.Println("Saving failure report:", err)
-					http.Error(w, "Internal server error", http.StatusInternalServerError)
-					return
-				}
-				pkt.Extra["goroutinesURL"] = url
-			}
-			message := sanitizeMessageLDB(r.Description)
-			pkt.Fingerprint = []string{message}
-
-			if err := sendReport(dsn, pkt, userIDFor(req)); err != nil {
-				log.Println("Failed to send failure report:", err)
-			} else {
-				log.Println("Sent failure report:", r.Description)
-				result = "success"
-			}
-		}
-	}
-}
-
-func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string, error) {
-	bs := make([]byte, len(data.Description)+len(data.Goroutines))
-	copy(bs, data.Description)
-	copy(bs[len(data.Description):], data.Goroutines)
-	id := fmt.Sprintf("%x", sha256.Sum256(bs))
-	path := fullPathCompressed(failureDir, id)
-	err := compressAndWrite(bs, path)
-	if err != nil {
-		return "", err
-	}
-	return reportServer + path, nil
-}
-
-type ignorePatterns struct {
-	patterns []*regexp.Regexp
-}
-
-func loadIgnorePatterns(path string) (*ignorePatterns, error) {
-	bs, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-
-	var patterns []*regexp.Regexp
-	for _, line := range strings.Split(string(bs), "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" {
-			continue
-		}
-		re, err := regexp.Compile(line)
-		if err != nil {
-			return nil, err
-		}
-		patterns = append(patterns, re)
-	}
-
-	log.Printf("Loaded %d ignore patterns", len(patterns))
-	return &ignorePatterns{patterns: patterns}, nil
-}
-
-func (i *ignorePatterns) match(report []byte) bool {
-	if i == nil {
-		return false
-	}
-	for _, re := range i.patterns {
-		if re.Match(report) {
-			return true
-		}
-	}
-	return false
-}

cmd/infra/stcrashreceiver/metrics.go

@@ -1,40 +0,0 @@
-// Copyright (C) 2023 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-var (
-	metricCrashReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "crashreceiver",
-		Name:      "crash_reports_total",
-	}, []string{"result"})
-	metricFailureReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "crashreceiver",
-		Name:      "failure_reports_total",
-	}, []string{"result"})
-	metricDiskstoreFilesTotal = promauto.NewGauge(prometheus.GaugeOpts{
-		Namespace: "syncthing",
-		Subsystem: "crashreceiver",
-		Name:      "diskstore_files_total",
-	})
-	metricDiskstoreBytesTotal = promauto.NewGauge(prometheus.GaugeOpts{
-		Namespace: "syncthing",
-		Subsystem: "crashreceiver",
-		Name:      "diskstore_bytes_total",
-	})
-	metricDiskstoreOldestAgeSeconds = promauto.NewGauge(prometheus.GaugeOpts{
-		Namespace: "syncthing",
-		Subsystem: "crashreceiver",
-		Name:      "diskstore_oldest_age_seconds",
-	})
-)

cmd/infra/stcrashreceiver/stcrashreceiver.go

@@ -1,133 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"io"
-	"log"
-	"net/http"
-	"path"
-	"strings"
-	"sync"
-)
-
-type crashReceiver struct {
-	store  *diskStore
-	sentry *sentryService
-	ignore *ignorePatterns
-
-	ignoredMut sync.RWMutex
-	ignored    map[string]struct{}
-}
-
-func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	// The final path component should be a SHA256 hash in hex, so 64 hex
-	// characters. We don't care about case on the request but use lower
-	// case internally.
-	reportID := strings.ToLower(path.Base(req.URL.Path))
-	if len(reportID) != 64 {
-		http.Error(w, "Bad request", http.StatusBadRequest)
-		return
-	}
-	for _, c := range reportID {
-		if c >= 'a' && c <= 'f' {
-			continue
-		}
-		if c >= '0' && c <= '9' {
-			continue
-		}
-		http.Error(w, "Bad request", http.StatusBadRequest)
-		return
-	}
-
-	switch req.Method {
-	case http.MethodGet:
-		r.serveGet(reportID, w, req)
-	case http.MethodHead:
-		r.serveHead(reportID, w, req)
-	case http.MethodPut:
-		r.servePut(reportID, w, req)
-	default:
-		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-
-// serveGet responds to GET requests by serving the uncompressed report.
-func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http.Request) {
-	bs, err := r.store.Get(reportID)
-	if err != nil {
-		http.Error(w, "Not found", http.StatusNotFound)
-		return
-	}
-	w.Write(bs)
-}
-
-// serveHead responds to HEAD requests by checking if the named report
-// already exists in the system.
-func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) {
-	r.ignoredMut.RLock()
-	_, ignored := r.ignored[reportID]
-	r.ignoredMut.RUnlock()
-	if ignored {
-		return // found
-	}
-	if !r.store.Exists(reportID) {
-		http.Error(w, "Not found", http.StatusNotFound)
-	}
-}
-
-// servePut accepts and stores the given report.
-func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *http.Request) {
-	result := "receive_failure"
-	defer func() {
-		metricCrashReportsTotal.WithLabelValues(result).Inc()
-	}()
-
-	r.ignoredMut.RLock()
-	_, ignored := r.ignored[reportID]
-	r.ignoredMut.RUnlock()
-	if ignored {
-		result = "ignored_cached"
-		io.Copy(io.Discard, req.Body)
-		return // found
-	}
-
-	// Read at most maxRequestSize of report data.
-	log.Println("Receiving report", reportID)
-	lr := io.LimitReader(req.Body, maxRequestSize)
-	bs, err := io.ReadAll(lr)
-	if err != nil {
-		log.Println("Reading report:", err)
-		http.Error(w, "Internal server error", http.StatusInternalServerError)
-		return
-	}
-
-	if r.ignore.match(bs) {
-		r.ignoredMut.Lock()
-		if r.ignored == nil {
-			r.ignored = make(map[string]struct{})
-		}
-		r.ignored[reportID] = struct{}{}
-		r.ignoredMut.Unlock()
-		result = "ignored"
-		return
-	}
-
-	result = "success"
-
-	// Store the report
-	if !r.store.Put(reportID, bs) {
-		log.Println("Failed to store report (queue full):", reportID)
-		result = "queue_failure"
-	}
-
-	// Send the report to Sentry
-	if !r.sentry.Send(reportID, userIDFor(req), bs) {
-		log.Println("Failed to send report to sentry (queue full):", reportID)
-		result = "sentry_failure"
-	}
-}

cmd/infra/stcrashreceiver/util.go

@@ -1,56 +0,0 @@
-// Copyright (C) 2021 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"crypto/sha256"
-	"fmt"
-	"net"
-	"net/http"
-	"os"
-	"path/filepath"
-	"time"
-)
-
-// userIDFor returns a string we can use as the user ID for the purpose of
-// counting affected users. It's the truncated hash of a salt, the user
-// remote IP, and the current month.
-func userIDFor(req *http.Request) string {
-	addr := req.RemoteAddr
-	if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
-		addr = fwd
-	}
-	if host, _, err := net.SplitHostPort(addr); err == nil {
-		addr = host
-	}
-	now := time.Now().Format("200601")
-	salt := "stcrashreporter"
-	hash := sha256.Sum256([]byte(salt + addr + now))
-	return fmt.Sprintf("%x", hash[:8])
-}
-
-// 01234567890abcdef... => 01/23
-func dirFor(base string) string {
-	return filepath.Join(base[0:2], base[2:4])
-}
-
-func fullPathCompressed(root, reportID string) string {
-	return filepath.Join(root, dirFor(reportID), reportID) + ".gz"
-}
-
-func compressAndWrite(bs []byte, fullPath string) error {
-	// Compress the report for storage
-	buf := new(bytes.Buffer)
-	gw := gzip.NewWriter(buf)
-	_, _ = gw.Write(bs) // can't fail
-	gw.Close()
-
-	// Create an output file with the compressed report
-	return os.WriteFile(fullPath, buf.Bytes(), 0o644)
-}

cmd/infra/stupgrades/main.go

@@ -1,351 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"log/slog"
-	"net"
-	"net/http"
-	"os"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/alecthomas/kong"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
-	"github.com/syncthing/syncthing/lib/httpcache"
-	"github.com/syncthing/syncthing/lib/upgrade"
-)
-
-type cli struct {
-	Listen        string        `default:":8080" help:"Listen address"`
-	MetricsListen string        `default:":8082" help:"Listen address for metrics"`
-	URL           string        `short:"u" default:"https://api.github.com/repos/syncthing/syncthing/releases?per_page=25" help:"GitHub releases url"`
-	Forward       []string      `short:"f" help:"Forwarded pages, format: /path->https://example/com/url"`
-	CacheTime     time.Duration `default:"15m" help:"Cache time"`
-}
-
-func main() {
-	var params cli
-	kong.Parse(&params)
-	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
-		Level: slog.LevelInfo,
-	})))
-
-	if err := server(&params); err != nil {
-		fmt.Printf("Error: %v\n", err)
-		os.Exit(1)
-	}
-}
-
-func server(params *cli) error {
-	if params.MetricsListen != "" {
-		mux := http.NewServeMux()
-		mux.Handle("/metrics", promhttp.Handler())
-		metricsListen, err := net.Listen("tcp", params.MetricsListen)
-		if err != nil {
-			return fmt.Errorf("metrics: %w", err)
-		}
-		slog.Info("Metrics listener started", "addr", params.MetricsListen)
-		go func() {
-			if err := http.Serve(metricsListen, mux); err != nil {
-				slog.Warn("Metrics server returned", "error", err)
-			}
-		}()
-	}
-
-	cache := &cachedReleases{url: params.URL}
-	if err := cache.Update(context.Background()); err != nil {
-		return fmt.Errorf("initial cache update: %w", err)
-	} else {
-		slog.Info("Initial cache update done")
-	}
-
-	go func() {
-		for range time.NewTicker(params.CacheTime).C {
-			slog.Info("Refreshing cached releases", "url", params.URL)
-			if err := cache.Update(context.Background()); err != nil {
-				slog.Error("Failed to refresh cached releases", "url", params.URL, "error", err)
-			}
-		}
-	}()
-
-	ghRels := &githubReleases{cache: cache}
-	mux := http.NewServeMux()
-	mux.HandleFunc("/ping", ghRels.servePing)
-	mux.HandleFunc("/meta.json", ghRels.serveReleases)
-
-	for _, fwd := range params.Forward {
-		path, url, ok := strings.Cut(fwd, "->")
-		if !ok {
-			return fmt.Errorf("invalid forward: %q", fwd)
-		}
-		slog.Info("Forwarding", "from", path, "to", url)
-		name := strings.ReplaceAll(path, "/", "_")
-		mux.Handle(path, httpcache.SinglePath(&proxy{name: name, url: url}, params.CacheTime))
-	}
-
-	srv := &http.Server{
-		Addr:         params.Listen,
-		Handler:      mux,
-		ReadTimeout:  5 * time.Second,
-		WriteTimeout: 10 * time.Second,
-	}
-	srv.SetKeepAlivesEnabled(false)
-
-	srvListener, err := net.Listen("tcp", params.Listen)
-	if err != nil {
-		return fmt.Errorf("listen: %w", err)
-	}
-	slog.Info("Main listener started", "addr", params.Listen)
-
-	return srv.Serve(srvListener)
-}
-
-type githubReleases struct {
-	cache *cachedReleases
-}
-
-func (p *githubReleases) servePing(w http.ResponseWriter, req *http.Request) {
-	rels := p.cache.Releases()
-
-	if len(rels) == 0 {
-		http.Error(w, "No releases available", http.StatusServiceUnavailable)
-		return
-	}
-
-	w.Header().Set("Syncthing-Num-Releases", strconv.Itoa(len(rels)))
-	w.WriteHeader(http.StatusOK)
-}
-
-func (p *githubReleases) serveReleases(w http.ResponseWriter, req *http.Request) {
-	rels := p.cache.Releases()
-
-	ua := req.Header.Get("User-Agent")
-	osv := req.Header.Get("Syncthing-Os-Version")
-	if ua != "" && osv != "" {
-		// We should determine the compatibility of the releases.
-		rels = filterForCompabitility(rels, ua, osv)
-	} else {
-		metricFilterCalls.WithLabelValues("no-ua-or-osversion").Inc()
-	}
-
-	rels = filterForLatest(rels)
-
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	w.Header().Set("Access-Control-Allow-Origin", "*")
-	w.Header().Set("Access-Control-Allow-Methods", "GET")
-	w.Header().Set("Cache-Control", "public, max-age=900")
-	w.Header().Set("Vary", "User-Agent, Syncthing-Os-Version")
-	_ = json.NewEncoder(w).Encode(rels)
-
-	metricUpgradeChecks.Inc()
-}
-
-type proxy struct {
-	name string
-	url  string
-}
-
-func (p *proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	req, err := http.NewRequestWithContext(req.Context(), http.MethodGet, p.url, nil)
-	if err != nil {
-		metricHTTPRequests.WithLabelValues(p.name, "error").Inc()
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		metricHTTPRequests.WithLabelValues(p.name, "error").Inc()
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-	defer resp.Body.Close()
-	metricHTTPRequests.WithLabelValues(p.name, "success").Inc()
-
-	ct := resp.Header.Get("Content-Type")
-	w.Header().Set("Content-Type", ct)
-	if resp.StatusCode == http.StatusOK {
-		w.Header().Set("Cache-Control", "public, max-age=900")
-		w.Header().Set("Access-Control-Allow-Origin", "*")
-		w.Header().Set("Access-Control-Allow-Methods", "GET")
-	}
-	w.WriteHeader(resp.StatusCode)
-	if strings.HasPrefix(ct, "application/json") {
-		// Special JSON handling; clean it up a bit.
-		var v interface{}
-		if err := json.NewDecoder(resp.Body).Decode(&v); err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-		_ = json.NewEncoder(w).Encode(v)
-	} else {
-		_, _ = io.Copy(w, resp.Body)
-	}
-}
-
-// filterForLatest returns the latest stable and prerelease only. If the
-// stable version is newer (comes first in the list) there is no need to go
-// looking for a prerelease at all.
-func filterForLatest(rels []upgrade.Release) []upgrade.Release {
-	var filtered []upgrade.Release
-	var havePre bool
-	for _, rel := range rels {
-		if !rel.Prerelease {
-			// We found a stable version, we're good now.
-			filtered = append(filtered, rel)
-			break
-		}
-		if rel.Prerelease && !havePre {
-			// We remember the first prerelease we find.
-			filtered = append(filtered, rel)
-			havePre = true
-		}
-	}
-	return filtered
-}
-
-var userAgentOSArchExp = regexp.MustCompile(`^syncthing.*\(.+ (\w+)-(\w+)\)$`)
-
-func filterForCompabitility(rels []upgrade.Release, ua, osv string) []upgrade.Release {
-	osArch := userAgentOSArchExp.FindStringSubmatch(ua)
-	if len(osArch) != 3 {
-		metricFilterCalls.WithLabelValues("bad-os-arch").Inc()
-		return rels
-	}
-	os := osArch[1]
-
-	var filtered []upgrade.Release
-	for _, rel := range rels {
-		if rel.Compatibility == nil {
-			// No requirements means it's compatible with everything.
-			filtered = append(filtered, rel)
-			continue
-		}
-
-		req, ok := rel.Compatibility.Requirements[os]
-		if !ok {
-			// No entry for the current OS means it's compatible.
-			filtered = append(filtered, rel)
-			continue
-		}
-
-		if upgrade.CompareVersions(osv, req) >= 0 {
-			filtered = append(filtered, rel)
-			continue
-		}
-	}
-
-	if len(filtered) != len(rels) {
-		metricFilterCalls.WithLabelValues("filtered").Inc()
-	} else {
-		metricFilterCalls.WithLabelValues("unchanged").Inc()
-	}
-
-	return filtered
-}
-
-type cachedReleases struct {
-	url     string
-	mut     sync.RWMutex
-	current []upgrade.Release
-}
-
-func (c *cachedReleases) Releases() []upgrade.Release {
-	c.mut.RLock()
-	defer c.mut.RUnlock()
-	return c.current
-}
-
-func (c *cachedReleases) Update(ctx context.Context) error {
-	rels, err := fetchGithubReleases(ctx, c.url)
-	if err != nil {
-		return err
-	}
-	c.mut.Lock()
-	c.current = rels
-	c.mut.Unlock()
-	return nil
-}
-
-func fetchGithubReleases(ctx context.Context, url string) ([]upgrade.Release, error) {
-	req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, url, nil)
-	if err != nil {
-		metricHTTPRequests.WithLabelValues("github-releases", "error").Inc()
-		return nil, err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		metricHTTPRequests.WithLabelValues("github-releases", "error").Inc()
-		return nil, err
-	}
-	defer resp.Body.Close()
-	var rels []upgrade.Release
-	if err := json.NewDecoder(resp.Body).Decode(&rels); err != nil {
-		metricHTTPRequests.WithLabelValues("github-releases", "error").Inc()
-		return nil, err
-	}
-	metricHTTPRequests.WithLabelValues("github-releases", "success").Inc()
-
-	// Move the URL used for browser downloads to the URL field, and remove
-	// the browser URL field. This avoids going via the GitHub API for
-	// downloads, since Syncthing uses the URL field.
-	for _, rel := range rels {
-		for j, asset := range rel.Assets {
-			rel.Assets[j].URL = asset.BrowserURL
-			rel.Assets[j].BrowserURL = ""
-		}
-	}
-
-	addReleaseCompatibility(ctx, rels)
-
-	sort.Sort(upgrade.SortByRelease(rels))
-	return rels, nil
-}
-
-func addReleaseCompatibility(ctx context.Context, rels []upgrade.Release) {
-	for i := range rels {
-		rel := &rels[i]
-		for i, asset := range rel.Assets {
-			if asset.Name != "compat.json" {
-				continue
-			}
-
-			// Load compat.json into the Compatibility field
-			req, err := http.NewRequestWithContext(ctx, http.MethodGet, asset.URL, nil)
-			if err != nil {
-				metricHTTPRequests.WithLabelValues("compat-json", "error").Inc()
-				break
-			}
-			resp, err := http.DefaultClient.Do(req)
-			if err != nil {
-				metricHTTPRequests.WithLabelValues("compat-json", "error").Inc()
-				break
-			}
-			if resp.StatusCode != http.StatusOK {
-				metricHTTPRequests.WithLabelValues("compat-json", "error").Inc()
-				resp.Body.Close()
-				break
-			}
-			_ = json.NewDecoder(io.LimitReader(resp.Body, 10<<10)).Decode(&rel.Compatibility)
-			metricHTTPRequests.WithLabelValues("compat-json", "success").Inc()
-			resp.Body.Close()
-
-			// Remove compat.json from the asset list since it's been processed
-			rel.Assets = append(rel.Assets[:i], rel.Assets[i+1:]...)
-			break
-		}
-	}
-}

cmd/infra/stupgrades/metrics.go

@@ -1,30 +0,0 @@
-// Copyright (C) 2024 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-var (
-	metricUpgradeChecks = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "upgrade",
-		Name:      "metadata_requests",
-	})
-	metricFilterCalls = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "upgrade",
-		Name:      "filter_calls",
-	}, []string{"result"})
-	metricHTTPRequests = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "upgrade",
-		Name:      "http_requests",
-	}, []string{"target", "result"})
-)

cmd/infra/ursrv/main.go

@@ -1,33 +0,0 @@
-// Copyright (C) 2023 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"log"
-	"log/slog"
-	"os"
-
-	"github.com/alecthomas/kong"
-	"github.com/syncthing/syncthing/cmd/infra/ursrv/serve"
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
-)
-
-type CLI struct {
-	Serve serve.CLI `cmd:"" default:""`
-}
-
-func main() {
-	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
-		Level: slog.LevelInfo,
-	})))
-
-	var cli CLI
-	ctx := kong.Parse(&cli)
-	if err := ctx.Run(); err != nil {
-		log.Fatalf("%s: %v", ctx.Command(), err)
-	}
-}

cmd/infra/ursrv/serve/metrics.go

@@ -1,46 +0,0 @@
-// Copyright (C) 2023 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package serve
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-var (
-	metricReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "ursrv_v2",
-		Name:      "incoming_reports_total",
-	}, []string{"result"})
-	metricsCollectsTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "ursrv_v2",
-		Name:      "collects_total",
-	})
-	metricsCollectSecondsTotal = promauto.NewCounter(prometheus.CounterOpts{
-		Namespace: "syncthing",
-		Subsystem: "ursrv_v2",
-		Name:      "collect_seconds_total",
-	})
-	metricsCollectSecondsLast = promauto.NewGauge(prometheus.GaugeOpts{
-		Namespace: "syncthing",
-		Subsystem: "ursrv_v2",
-		Name:      "collect_seconds_last",
-	})
-	metricsWriteSecondsLast = promauto.NewGauge(prometheus.GaugeOpts{
-		Namespace: "syncthing",
-		Subsystem: "ursrv_v2",
-		Name:      "write_seconds_last",
-	})
-)
-
-func init() {
-	metricReportsTotal.WithLabelValues("fail")
-	metricReportsTotal.WithLabelValues("replace")
-	metricReportsTotal.WithLabelValues("accept")
-}

cmd/infra/ursrv/serve/prometheus.go

@@ -1,314 +0,0 @@
-// Copyright (C) 2024 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package serve
-
-import (
-	"reflect"
-	"slices"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/syncthing/syncthing/lib/ur/contract"
-)
-
-const namePrefix = "syncthing_usage_"
-
-type metricsSet struct {
-	srv *server
-
-	gauges         map[string]prometheus.Gauge
-	gaugeVecs      map[string]*prometheus.GaugeVec
-	gaugeVecLabels map[string][]string
-	summaries      map[string]*metricSummary
-
-	collectMut    sync.Mutex
-	collectCutoff time.Duration
-}
-
-func newMetricsSet(srv *server) *metricsSet {
-	s := &metricsSet{
-		srv:            srv,
-		gauges:         make(map[string]prometheus.Gauge),
-		gaugeVecs:      make(map[string]*prometheus.GaugeVec),
-		gaugeVecLabels: make(map[string][]string),
-		summaries:      make(map[string]*metricSummary),
-		collectCutoff:  -24 * time.Hour,
-	}
-
-	var initForType func(reflect.Type)
-	initForType = func(t reflect.Type) {
-		for i := 0; i < t.NumField(); i++ {
-			field := t.Field(i)
-			if field.Type.Kind() == reflect.Struct {
-				initForType(field.Type)
-				continue
-			}
-			name, typ, label := fieldNameTypeLabel(field)
-			sname, labels := nameConstLabels(name)
-			switch typ {
-			case "gauge":
-				s.gauges[name] = prometheus.NewGauge(prometheus.GaugeOpts{
-					Name:        namePrefix + sname,
-					ConstLabels: labels,
-				})
-			case "summary":
-				s.summaries[name] = newMetricSummary(namePrefix+sname, nil, labels)
-			case "gaugeVec":
-				s.gaugeVecLabels[name] = append(s.gaugeVecLabels[name], label)
-			case "summaryVec":
-				s.summaries[name] = newMetricSummary(namePrefix+sname, []string{label}, labels)
-			}
-		}
-	}
-	initForType(reflect.ValueOf(contract.Report{}).Type())
-
-	for name, labels := range s.gaugeVecLabels {
-		s.gaugeVecs[name] = prometheus.NewGaugeVec(prometheus.GaugeOpts{
-			Name: namePrefix + name,
-		}, labels)
-	}
-
-	return s
-}
-
-func fieldNameTypeLabel(rf reflect.StructField) (string, string, string) {
-	metric := rf.Tag.Get("metric")
-	name, typ, ok := strings.Cut(metric, ",")
-	if !ok {
-		return "", "", ""
-	}
-	gv, label, ok := strings.Cut(typ, ":")
-	if ok {
-		typ = gv
-	}
-	return name, typ, label
-}
-
-func nameConstLabels(name string) (string, prometheus.Labels) {
-	if name == "-" {
-		return "", nil
-	}
-	name, labels, ok := strings.Cut(name, "{")
-	if !ok {
-		return name, nil
-	}
-	lls := strings.Split(labels[:len(labels)-1], ",")
-	m := make(map[string]string)
-	for _, l := range lls {
-		k, v, _ := strings.Cut(l, "=")
-		m[k] = v
-	}
-	return name, m
-}
-
-func (s *metricsSet) addReport(r *contract.Report) {
-	gaugeVecs := make(map[string][]string)
-	s.addReportStruct(reflect.ValueOf(r).Elem(), gaugeVecs)
-	for name, lv := range gaugeVecs {
-		s.gaugeVecs[name].WithLabelValues(lv...).Add(1)
-	}
-}
-
-func (s *metricsSet) addReportStruct(v reflect.Value, gaugeVecs map[string][]string) {
-	t := v.Type()
-	for i := 0; i < v.NumField(); i++ {
-		field := v.Field(i)
-		if field.Kind() == reflect.Struct {
-			s.addReportStruct(field, gaugeVecs)
-			continue
-		}
-
-		name, typ, label := fieldNameTypeLabel(t.Field(i))
-		switch typ {
-		case "gauge":
-			switch v := field.Interface().(type) {
-			case int:
-				s.gauges[name].Add(float64(v))
-			case string:
-				s.gaugeVecs[name].WithLabelValues(v).Add(1)
-			case bool:
-				if v {
-					s.gauges[name].Add(1)
-				}
-			}
-		case "gaugeVec":
-			var labelValue string
-			switch v := field.Interface().(type) {
-			case string:
-				labelValue = v
-			case int:
-				labelValue = strconv.Itoa(v)
-			case map[string]int:
-				for k, v := range v {
-					labelValue = k
-					field.SetInt(int64(v))
-					break
-				}
-			}
-			if _, ok := gaugeVecs[name]; !ok {
-				gaugeVecs[name] = make([]string, len(s.gaugeVecLabels[name]))
-			}
-			for i, l := range s.gaugeVecLabels[name] {
-				if l == label {
-					gaugeVecs[name][i] = labelValue
-					break
-				}
-			}
-		case "summary", "summaryVec":
-			switch v := field.Interface().(type) {
-			case int:
-				s.summaries[name].Observe("", float64(v))
-			case float64:
-				s.summaries[name].Observe("", v)
-			case []int:
-				for _, v := range v {
-					s.summaries[name].Observe("", float64(v))
-				}
-			case map[string]int:
-				for k, v := range v {
-					if k == "" {
-						// avoid empty string labels as those are the sign
-						// of a non-vec summary
-						k = "unknown"
-					}
-					s.summaries[name].Observe(k, float64(v))
-				}
-			}
-		}
-	}
-}
-
-func (s *metricsSet) Describe(c chan<- *prometheus.Desc) {
-	for _, g := range s.gauges {
-		g.Describe(c)
-	}
-	for _, g := range s.gaugeVecs {
-		g.Describe(c)
-	}
-	for _, g := range s.summaries {
-		g.Describe(c)
-	}
-}
-
-func (s *metricsSet) Collect(c chan<- prometheus.Metric) {
-	s.collectMut.Lock()
-	defer s.collectMut.Unlock()
-
-	t0 := time.Now()
-	defer func() {
-		dur := time.Since(t0).Seconds()
-		metricsCollectSecondsLast.Set(dur)
-		metricsCollectSecondsTotal.Add(dur)
-		metricsCollectsTotal.Inc()
-	}()
-
-	for _, g := range s.gauges {
-		g.Set(0)
-	}
-	for _, g := range s.gaugeVecs {
-		g.Reset()
-	}
-	for _, g := range s.summaries {
-		g.Reset()
-	}
-
-	cutoff := time.Now().Add(s.collectCutoff)
-	s.srv.reports.Range(func(key string, r *contract.Report) bool {
-		if s.collectCutoff < 0 && r.Received.Before(cutoff) {
-			s.srv.reports.Delete(key)
-			return true
-		}
-		s.addReport(r)
-		return true
-	})
-
-	for _, g := range s.gauges {
-		c <- g
-	}
-	for _, g := range s.gaugeVecs {
-		g.Collect(c)
-	}
-	for _, g := range s.summaries {
-		g.Collect(c)
-	}
-}
-
-type metricSummary struct {
-	name   string
-	values map[string][]float64
-	zeroes map[string]int
-
-	qDesc     *prometheus.Desc
-	countDesc *prometheus.Desc
-	sumDesc   *prometheus.Desc
-	zDesc     *prometheus.Desc
-}
-
-func newMetricSummary(name string, labels []string, constLabels prometheus.Labels) *metricSummary {
-	return &metricSummary{
-		name:      name,
-		values:    make(map[string][]float64),
-		zeroes:    make(map[string]int),
-		qDesc:     prometheus.NewDesc(name, "", append(labels, "quantile"), constLabels),
-		countDesc: prometheus.NewDesc(name+"_nonzero_count", "", labels, constLabels),
-		sumDesc:   prometheus.NewDesc(name+"_sum", "", labels, constLabels),
-		zDesc:     prometheus.NewDesc(name+"_zero_count", "", labels, constLabels),
-	}
-}
-
-func (q *metricSummary) Observe(labelValue string, v float64) {
-	if v == 0 {
-		q.zeroes[labelValue]++
-		return
-	}
-	q.values[labelValue] = append(q.values[labelValue], v)
-}
-
-func (q *metricSummary) Describe(c chan<- *prometheus.Desc) {
-	c <- q.qDesc
-	c <- q.countDesc
-	c <- q.sumDesc
-	c <- q.zDesc
-}
-
-func (q *metricSummary) Collect(c chan<- prometheus.Metric) {
-	for lv, vs := range q.values {
-		var labelVals []string
-		if lv != "" {
-			labelVals = []string{lv}
-		}
-
-		c <- prometheus.MustNewConstMetric(q.countDesc, prometheus.GaugeValue, float64(len(vs)), labelVals...)
-		c <- prometheus.MustNewConstMetric(q.zDesc, prometheus.GaugeValue, float64(q.zeroes[lv]), labelVals...)
-
-		var sum float64
-		for _, v := range vs {
-			sum += v
-		}
-		c <- prometheus.MustNewConstMetric(q.sumDesc, prometheus.GaugeValue, sum, labelVals...)
-
-		if len(vs) == 0 {
-			return
-		}
-
-		slices.Sort(vs)
-		c <- prometheus.MustNewConstMetric(q.qDesc, prometheus.GaugeValue, vs[0], append(labelVals, "0")...)
-		c <- prometheus.MustNewConstMetric(q.qDesc, prometheus.GaugeValue, vs[len(vs)*5/100], append(labelVals, "0.05")...)
-		c <- prometheus.MustNewConstMetric(q.qDesc, prometheus.GaugeValue, vs[len(vs)/2], append(labelVals, "0.5")...)
-		c <- prometheus.MustNewConstMetric(q.qDesc, prometheus.GaugeValue, vs[len(vs)*9/10], append(labelVals, "0.9")...)
-		c <- prometheus.MustNewConstMetric(q.qDesc, prometheus.GaugeValue, vs[len(vs)*95/100], append(labelVals, "0.95")...)
-		c <- prometheus.MustNewConstMetric(q.qDesc, prometheus.GaugeValue, vs[len(vs)-1], append(labelVals, "1")...)
-	}
-}
-
-func (q *metricSummary) Reset() {
-	clear(q.values)
-	clear(q.zeroes)
-}

cmd/infra/ursrv/serve/serve.go

@@ -1,406 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package serve
-
-import (
-	"bufio"
-	"compress/gzip"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"log/slog"
-	"net"
-	"net/http"
-	_ "net/http/pprof"
-	"os"
-	"regexp"
-	"strings"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/puzpuzpuz/xsync/v3"
-	"github.com/syncthing/syncthing/lib/build"
-	"github.com/syncthing/syncthing/lib/geoip"
-	"github.com/syncthing/syncthing/lib/s3"
-	"github.com/syncthing/syncthing/lib/ur/contract"
-)
-
-type CLI struct {
-	Listen          string        `env:"UR_LISTEN" help:"Usage reporting & metrics endpoint listen address" default:"0.0.0.0:8080"`
-	ListenInternal  string        `env:"UR_LISTEN_INTERNAL" help:"Internal metrics endpoint listen address" default:"0.0.0.0:8082"`
-	GeoIPLicenseKey string        `env:"UR_GEOIP_LICENSE_KEY"`
-	GeoIPAccountID  int           `env:"UR_GEOIP_ACCOUNT_ID"`
-	DumpFile        string        `env:"UR_DUMP_FILE" default:"reports.jsons.gz"`
-	DumpInterval    time.Duration `env:"UR_DUMP_INTERVAL" default:"5m"`
-
-	S3Endpoint    string `name:"s3-endpoint" hidden:"true" env:"UR_S3_ENDPOINT"`
-	S3Region      string `name:"s3-region" hidden:"true" env:"UR_S3_REGION"`
-	S3Bucket      string `name:"s3-bucket" hidden:"true" env:"UR_S3_BUCKET"`
-	S3AccessKeyID string `name:"s3-access-key-id" hidden:"true" env:"UR_S3_ACCESS_KEY_ID"`
-	S3SecretKey   string `name:"s3-secret-key" hidden:"true" env:"UR_S3_SECRET_KEY"`
-}
-
-var (
-	compilerRe         = regexp.MustCompile(`\(([A-Za-z0-9()., -]+) \w+-\w+(?:| android| default)\) ([\w@.-]+)`)
-	knownDistributions = []distributionMatch{
-		// Maps well known builders to the official distribution method that
-		// they represent
-
-		{regexp.MustCompile(`\steamcity@build\.syncthing\.net`), "GitHub"},
-		{regexp.MustCompile(`\sjenkins@build\.syncthing\.net`), "GitHub"},
-		{regexp.MustCompile(`\sbuilder@github\.syncthing\.net`), "GitHub"},
-
-		{regexp.MustCompile(`\sdeb@build\.syncthing\.net`), "APT"},
-		{regexp.MustCompile(`\sdebian@github\.syncthing\.net`), "APT"},
-
-		{regexp.MustCompile(`\sdocker@syncthing\.net`), "Docker Hub"},
-		{regexp.MustCompile(`\sdocker@build.syncthing\.net`), "Docker Hub"},
-		{regexp.MustCompile(`\sdocker@github.syncthing\.net`), "Docker Hub"},
-
-		{regexp.MustCompile(`\sandroid-builder@github\.syncthing\.net`), "Google Play"},
-		{regexp.MustCompile(`\sandroid-.*teamcity@build\.syncthing\.net`), "Google Play"},
-
-		{regexp.MustCompile(`\sandroid-.*vagrant@basebox-stretch64`), "F-Droid"},
-		{regexp.MustCompile(`\svagrant@bullseye`), "F-Droid"},
-		{regexp.MustCompile(`\svagrant@bookworm`), "F-Droid"},
-
-		{regexp.MustCompile(`Anwender@NET2017`), "Syncthing-Fork (3rd party)"},
-
-		{regexp.MustCompile(`\sbuilduser@(archlinux|svetlemodry)`), "Arch (3rd party)"},
-		{regexp.MustCompile(`\ssyncthing@archlinux`), "Arch (3rd party)"},
-		{regexp.MustCompile(`@debian`), "Debian (3rd party)"},
-		{regexp.MustCompile(`@fedora`), "Fedora (3rd party)"},
-		{regexp.MustCompile(`\sbrew@`), "Homebrew (3rd party)"},
-		{regexp.MustCompile(`\sroot@buildkitsandbox`), "LinuxServer.io (3rd party)"},
-		{regexp.MustCompile(`\sports@freebsd`), "FreeBSD (3rd party)"},
-		{regexp.MustCompile(`\snix@nix`), "Nix (3rd party)"},
-		{regexp.MustCompile(`.`), "Others"},
-	}
-)
-
-type distributionMatch struct {
-	matcher      *regexp.Regexp
-	distribution string
-}
-
-func (cli *CLI) Run() error {
-	slog.Info("Starting", "version", build.Version)
-
-	// Listening
-
-	urListener, err := net.Listen("tcp", cli.Listen)
-	if err != nil {
-		slog.Error("Failed to listen (usage reports)", "error", err)
-		return err
-	}
-	slog.Info("Listening (usage reports)", "address", urListener.Addr())
-
-	internalListener, err := net.Listen("tcp", cli.ListenInternal)
-	if err != nil {
-		slog.Error("Failed to listen (internal)", "error", err)
-		return err
-	}
-	slog.Info("Listening (internal)", "address", internalListener.Addr())
-
-	var geo *geoip.Provider
-	if cli.GeoIPAccountID != 0 && cli.GeoIPLicenseKey != "" {
-		geo, err = geoip.NewGeoLite2CityProvider(context.Background(), cli.GeoIPAccountID, cli.GeoIPLicenseKey, os.TempDir())
-		if err != nil {
-			slog.Error("Failed to load GeoIP", "error", err)
-			return err
-		}
-		go geo.Serve(context.TODO())
-	}
-
-	// s3
-
-	var s3sess *s3.Session
-	if cli.S3Endpoint != "" {
-		s3sess, err = s3.NewSession(cli.S3Endpoint, cli.S3Region, cli.S3Bucket, cli.S3AccessKeyID, cli.S3SecretKey)
-		if err != nil {
-			slog.Error("Failed to create S3 session", "error", err)
-			return err
-		}
-	}
-
-	if _, err := os.Stat(cli.DumpFile); err != nil && s3sess != nil {
-		if err := cli.downloadDumpFile(s3sess); err != nil {
-			slog.Error("Failed to download dump file", "error", err)
-		}
-	}
-
-	// server
-
-	srv := &server{
-		geo:     geo,
-		reports: xsync.NewMapOf[string, *contract.Report](),
-	}
-
-	if fd, err := os.Open(cli.DumpFile); err == nil {
-		gr, err := gzip.NewReader(fd)
-		if err == nil {
-			srv.load(gr)
-		}
-		fd.Close()
-	}
-
-	go func() {
-		for range time.Tick(cli.DumpInterval) {
-			if err := cli.saveDumpFile(srv, s3sess); err != nil {
-				slog.Error("Failed to write dump file", "error", err)
-			}
-		}
-	}()
-
-	// The internal metrics endpoint just serves metrics about what the
-	// server is doing.
-
-	http.Handle("/metrics", promhttp.Handler())
-
-	internalSrv := http.Server{
-		ReadTimeout:  5 * time.Second,
-		WriteTimeout: 15 * time.Second,
-	}
-	go internalSrv.Serve(internalListener)
-
-	// New external metrics endpoint accepts reports from clients and serves
-	// aggregated usage reporting metrics.
-
-	ms := newMetricsSet(srv)
-	reg := prometheus.NewRegistry()
-	reg.MustRegister(ms)
-
-	mux := http.NewServeMux()
-	mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
-	mux.HandleFunc("/newdata", srv.handleNewData)
-	mux.HandleFunc("/ping", srv.handlePing)
-
-	metricsSrv := http.Server{
-		ReadTimeout:  5 * time.Second,
-		WriteTimeout: 15 * time.Second,
-		Handler:      mux,
-	}
-
-	slog.Info("Ready to serve")
-	return metricsSrv.Serve(urListener)
-}
-
-func (cli *CLI) downloadDumpFile(s3sess *s3.Session) error {
-	latestKey, err := s3sess.LatestKey()
-	if err != nil {
-		return fmt.Errorf("list latest S3 key: %w", err)
-	}
-	fd, err := os.Create(cli.DumpFile)
-	if err != nil {
-		return fmt.Errorf("create dump file: %w", err)
-	}
-	if err := s3sess.Download(fd, latestKey); err != nil {
-		_ = fd.Close()
-		return fmt.Errorf("download dump file: %w", err)
-	}
-	if err := fd.Close(); err != nil {
-		return fmt.Errorf("close dump file: %w", err)
-	}
-	slog.Info("Dump file downloaded", "key", latestKey)
-	return nil
-}
-
-func (cli *CLI) saveDumpFile(srv *server, s3sess *s3.Session) error {
-	fd, err := os.Create(cli.DumpFile + ".tmp")
-	if err != nil {
-		return fmt.Errorf("creating dump file: %w", err)
-	}
-	gw := gzip.NewWriter(fd)
-	if err := srv.save(gw); err != nil {
-		return fmt.Errorf("saving dump file: %w", err)
-	}
-	if err := gw.Close(); err != nil {
-		fd.Close()
-		return fmt.Errorf("closing gzip writer: %w", err)
-	}
-	if err := fd.Close(); err != nil {
-		return fmt.Errorf("closing dump file: %w", err)
-	}
-	if err := os.Rename(cli.DumpFile+".tmp", cli.DumpFile); err != nil {
-		return fmt.Errorf("renaming dump file: %w", err)
-	}
-	slog.Info("Dump file saved")
-
-	if s3sess != nil {
-		key := fmt.Sprintf("reports-%s.jsons.gz", time.Now().UTC().Format("2006-01-02"))
-		fd, err := os.Open(cli.DumpFile)
-		if err != nil {
-			return fmt.Errorf("opening dump file: %w", err)
-		}
-		if err := s3sess.Upload(fd, key); err != nil {
-			return fmt.Errorf("uploading dump file: %w", err)
-		}
-		_ = fd.Close()
-		slog.Info("Dump file uploaded")
-	}
-
-	return nil
-}
-
-type server struct {
-	geo     *geoip.Provider
-	reports *xsync.MapOf[string, *contract.Report]
-}
-
-func (s *server) handlePing(w http.ResponseWriter, r *http.Request) {
-}
-
-func (s *server) handleNewData(w http.ResponseWriter, r *http.Request) {
-	result := "fail"
-	defer func() {
-		// result is "accept" (new report), "replace" (existing report) or
-		// "fail"
-		metricReportsTotal.WithLabelValues(result).Inc()
-	}()
-
-	defer r.Body.Close()
-
-	if r.Method != http.MethodPost {
-		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
-		return
-	}
-
-	addr := r.Header.Get("X-Forwarded-For")
-	if addr != "" {
-		addr = strings.Split(addr, ", ")[0]
-	} else {
-		addr = r.RemoteAddr
-	}
-
-	if host, _, err := net.SplitHostPort(addr); err == nil {
-		addr = host
-	}
-
-	log := slog.With("addr", addr)
-
-	if net.ParseIP(addr) == nil {
-		addr = ""
-	}
-
-	var rep contract.Report
-
-	lr := &io.LimitedReader{R: r.Body, N: 40 * 1024}
-	bs, _ := io.ReadAll(lr)
-	if err := json.Unmarshal(bs, &rep); err != nil {
-		log.Error("Failed to decode JSON", "error", err)
-		http.Error(w, "JSON Decode Error", http.StatusInternalServerError)
-		return
-	}
-
-	rep.Received = time.Now()
-	rep.Date = rep.Received.UTC().Format("20060102")
-	rep.Address = addr
-
-	if err := rep.Validate(); err != nil {
-		log.Error("Failed to validate report", "error", err)
-		http.Error(w, "Validation Error", http.StatusInternalServerError)
-		return
-	}
-
-	if s.addReport(&rep) {
-		result = "replace"
-	} else {
-		result = "accept"
-	}
-}
-
-func (s *server) addReport(rep *contract.Report) bool {
-	if s.geo != nil {
-		if ip := net.ParseIP(rep.Address); ip != nil {
-			if city, err := s.geo.City(ip); err == nil {
-				rep.Country = city.Country.Names["en"]
-				rep.CountryCode = city.Country.IsoCode
-			}
-		}
-	}
-	if rep.Country == "" {
-		rep.Country = "Unknown"
-	}
-	if rep.CountryCode == "" {
-		rep.CountryCode = "ZZ"
-	}
-
-	rep.Version = transformVersion(rep.Version)
-	if strings.Contains(rep.Version, ".") {
-		split := strings.SplitN(rep.Version, ".", 3)
-		if len(split) == 3 {
-			rep.MajorVersion = strings.Join(split[:2], ".")
-		}
-	}
-	rep.OS, rep.Arch, _ = strings.Cut(rep.Platform, "-")
-
-	if m := compilerRe.FindStringSubmatch(rep.LongVersion); len(m) == 3 {
-		rep.Compiler = m[1]
-		rep.Builder = m[2]
-	}
-	for _, d := range knownDistributions {
-		if d.matcher.MatchString(rep.LongVersion) {
-			rep.Distribution = d.distribution
-			break
-		}
-	}
-
-	_, loaded := s.reports.LoadAndStore(rep.UniqueID, rep)
-	return loaded
-}
-
-func (s *server) save(w io.Writer) error {
-	bw := bufio.NewWriter(w)
-	enc := json.NewEncoder(bw)
-	var err error
-	s.reports.Range(func(k string, v *contract.Report) bool {
-		err = enc.Encode(v)
-		return err == nil
-	})
-	if err != nil {
-		return err
-	}
-	return bw.Flush()
-}
-
-func (s *server) load(r io.Reader) {
-	dec := json.NewDecoder(r)
-	s.reports.Clear()
-	for {
-		var rep contract.Report
-		if err := dec.Decode(&rep); errors.Is(err, io.EOF) {
-			break
-		} else if err != nil {
-			slog.Error("Failed to load record", "error", err)
-			break
-		}
-		s.addReport(&rep)
-	}
-	slog.Info("Loaded reports", "count", s.reports.Size())
-}
-
-var (
-	plusRe  = regexp.MustCompile(`(\+.*|[.-]dev\..*)$`)
-	plusStr = "-dev"
-)
-
-// transformVersion returns a version number formatted correctly, with all
-// development versions aggregated into one.
-func transformVersion(v string) string {
-	if v == "unknown-dev" {
-		return v
-	}
-	if !strings.HasPrefix(v, "v") {
-		v = "v" + v
-	}
-	v = plusRe.ReplaceAllString(v, plusStr)
-
-	return v
-}

cmd/stcompdirs/main.go

@@ -7,7 +7,6 @@
 package main
 
 import (
-	"crypto/sha256"
 	"errors"
 	"flag"
 	"fmt"
@@ -16,7 +15,7 @@ import (
 	"os"
 	"path/filepath"
 
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 func main() {

cmd/stcrashreceiver/main.go

@@ -0,0 +1,119 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Command stcrashreceiver is a trivial HTTP server that allows two things:
+//
+// - uploading files (crash reports) named like a SHA256 hash using a PUT request
+// - checking whether such file exists using a HEAD request
+//
+// Typically this should be deployed behind something that manages HTTPS.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/sha256"
+	"github.com/syncthing/syncthing/lib/ur"
+
+	raven "github.com/getsentry/raven-go"
+)
+
+const maxRequestSize = 1 << 20 // 1 MiB
+
+func main() {
+	dir := flag.String("dir", ".", "Directory to store reports in")
+	dsn := flag.String("dsn", "", "Sentry DSN")
+	listen := flag.String("listen", ":22039", "HTTP listen address")
+	flag.Parse()
+
+	mux := http.NewServeMux()
+
+	cr := &crashReceiver{
+		dir: *dir,
+		dsn: *dsn,
+	}
+	mux.Handle("/", cr)
+
+	if *dsn != "" {
+		mux.HandleFunc("/newcrash/failure", handleFailureFn(*dsn))
+	}
+
+	log.SetOutput(os.Stdout)
+	if err := http.ListenAndServe(*listen, mux); err != nil {
+		log.Fatalln("HTTP serve:", err)
+	}
+}
+
+func handleFailureFn(dsn string) func(w http.ResponseWriter, req *http.Request) {
+	return func(w http.ResponseWriter, req *http.Request) {
+		lr := io.LimitReader(req.Body, maxRequestSize)
+		bs, err := ioutil.ReadAll(lr)
+		req.Body.Close()
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+
+		var reports []ur.FailureReport
+		err = json.Unmarshal(bs, &reports)
+		if err != nil {
+			http.Error(w, err.Error(), 400)
+			return
+		}
+		if len(reports) == 0 {
+			// Shouldn't happen
+			log.Printf("Got zero failure reports")
+			return
+		}
+
+		version, err := parseVersion(reports[0].Version)
+		if err != nil {
+			http.Error(w, err.Error(), 400)
+			return
+		}
+		for _, r := range reports {
+			pkt := packet(version, "failure")
+			pkt.Message = r.Description
+			pkt.Extra = raven.Extra{
+				"count": r.Count,
+			}
+			message := sanitizeMessageLDB(r.Description)
+			pkt.Fingerprint = []string{message}
+
+			if err := sendReport(dsn, pkt, userIDFor(req)); err != nil {
+				log.Println("Failed to send failure report:", err)
+			} else {
+				log.Println("Sent failure report:", r.Description)
+			}
+		}
+	}
+}
+
+// userIDFor returns a string we can use as the user ID for the purpose of
+// counting affected users. It's the truncated hash of a salt, the user
+// remote IP, and the current month.
+func userIDFor(req *http.Request) string {
+	addr := req.RemoteAddr
+	if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
+		addr = fwd
+	}
+	if host, _, err := net.SplitHostPort(addr); err == nil {
+		addr = host
+	}
+	now := time.Now().Format("200601")
+	salt := "stcrashreporter"
+	hash := sha256.Sum256([]byte(salt + addr + now))
+	return fmt.Sprintf("%x", hash[:8])
+}

cmd/stcrashreceiver/sentry.go

@@ -8,17 +8,14 @@ package main
 
 import (
 	"bytes"
-	"context"
 	"errors"
-	"io"
-	"log"
+	"io/ioutil"
 	"regexp"
 	"strings"
 	"sync"
 
 	raven "github.com/getsentry/raven-go"
-	"github.com/maruel/panicparse/v2/stack"
-	"github.com/syncthing/syncthing/lib/build"
+	"github.com/maruel/panicparse/stack"
 )
 
 const reportServer = "https://crash.syncthing.net/report/"
@@ -34,45 +31,6 @@ var (
 	clientsMut sync.Mutex
 )
 
-type sentryService struct {
-	dsn   string
-	inbox chan sentryRequest
-}
-
-type sentryRequest struct {
-	reportID string
-	userID   string
-	data     []byte
-}
-
-func (s *sentryService) Serve(ctx context.Context) {
-	for {
-		select {
-		case req := <-s.inbox:
-			pkt, err := parseCrashReport(req.reportID, req.data)
-			if err != nil {
-				log.Println("Failed to parse crash report:", err)
-				continue
-			}
-			if err := sendReport(s.dsn, pkt, req.userID); err != nil {
-				log.Println("Failed to send crash report:", err)
-			}
-
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-func (s *sentryService) Send(reportID, userID string, data []byte) bool {
-	select {
-	case s.inbox <- sentryRequest{reportID, userID, data}:
-		return true
-	default:
-		return false
-	}
-}
-
 func sendReport(dsn string, pkt *raven.Packet, userID string) error {
 	pkt.Interfaces = append(pkt.Interfaces, &raven.User{ID: userID})
 
@@ -106,7 +64,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
 		return nil, errors.New("no first line")
 	}
 
-	version, err := build.ParseVersion(string(parts[0]))
+	version, err := parseVersion(string(parts[0]))
 	if err != nil {
 		return nil, err
 	}
@@ -135,21 +93,18 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
 	}
 
 	r := bytes.NewReader(report)
-	ctx, _, err := stack.ScanSnapshot(r, io.Discard, stack.DefaultOpts())
-	if err != nil && err != io.EOF {
+	ctx, err := stack.ParseDump(r, ioutil.Discard, false)
+	if err != nil {
 		return nil, err
 	}
-	if ctx == nil || len(ctx.Goroutines) == 0 {
-		return nil, errors.New("no goroutines found")
-	}
 
 	// Lock the source code loader to the version we are processing here.
-	if version.Commit != "" {
+	if version.commit != "" {
 		// We have a commit hash, so we know exactly which source to use
-		loader.LockWithVersion(version.Commit)
-	} else if strings.HasPrefix(version.Tag, "v") {
+		loader.LockWithVersion(version.commit)
+	} else if strings.HasPrefix(version.tag, "v") {
 		// Lets hope the tag is close enough
-		loader.LockWithVersion(version.Tag)
+		loader.LockWithVersion(version.tag)
 	} else {
 		// Last resort
 		loader.LockWithVersion("main")
@@ -161,7 +116,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
 		if gr.First {
 			trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
 			for i, sc := range gr.Stack.Calls {
-				trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name, sc.RemoteSrcPath, sc.Line, 3, nil)
+				trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name(), sc.SrcPath, sc.Line, 3, nil)
 			}
 			break
 		}
@@ -216,26 +171,89 @@ func crashReportFingerprint(message string) []string {
 	return []string{"{{ default }}", message}
 }
 
-func packet(version build.VersionParts, reportType string) *raven.Packet {
+// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]
+var longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
+
+type version struct {
+	version  string   // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
+	tag      string   // "v1.1.4-rc.1"
+	commit   string   // "6aaae618", blank when absent
+	codename string   // "Erbium Earthworm"
+	runtime  string   // "go1.12.5"
+	goos     string   // "darwin"
+	goarch   string   // "amd64"
+	builder  string   // "jb@kvin.kastelo.net"
+	extra    []string // "foo", "bar"
+}
+
+func (v version) environment() string {
+	if v.commit != "" {
+		return "Development"
+	}
+	if strings.Contains(v.tag, "-rc.") {
+		return "Candidate"
+	}
+	if strings.Contains(v.tag, "-") {
+		return "Beta"
+	}
+	return "Stable"
+}
+
+func parseVersion(line string) (version, error) {
+	m := longVersionRE.FindStringSubmatch(line)
+	if len(m) == 0 {
+		return version{}, errors.New("unintelligeble version string")
+	}
+
+	v := version{
+		version:  m[1],
+		codename: m[2],
+		runtime:  m[3],
+		goos:     m[4],
+		goarch:   m[5],
+		builder:  m[6],
+	}
+
+	parts := strings.Split(v.version, "+")
+	v.tag = parts[0]
+	if len(parts) > 1 {
+		fields := strings.Split(parts[1], "-")
+		if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
+			v.commit = fields[1][1:]
+		}
+	}
+
+	if len(m) >= 8 && m[7] != "" {
+		tags := strings.Split(m[7], ",")
+		for i := range tags {
+			tags[i] = strings.TrimSpace(tags[i])
+		}
+		v.extra = tags
+	}
+
+	return v, nil
+}
+
+func packet(version version, reportType string) *raven.Packet {
 	pkt := &raven.Packet{
 		Platform:    "go",
-		Release:     version.Tag,
-		Environment: version.Environment(),
+		Release:     version.tag,
+		Environment: version.environment(),
 		Tags: raven.Tags{
-			raven.Tag{Key: "version", Value: version.Version},
-			raven.Tag{Key: "tag", Value: version.Tag},
-			raven.Tag{Key: "codename", Value: version.Codename},
-			raven.Tag{Key: "runtime", Value: version.Runtime},
-			raven.Tag{Key: "goos", Value: version.GOOS},
-			raven.Tag{Key: "goarch", Value: version.GOARCH},
-			raven.Tag{Key: "builder", Value: version.Builder},
+			raven.Tag{Key: "version", Value: version.version},
+			raven.Tag{Key: "tag", Value: version.tag},
+			raven.Tag{Key: "codename", Value: version.codename},
+			raven.Tag{Key: "runtime", Value: version.runtime},
+			raven.Tag{Key: "goos", Value: version.goos},
+			raven.Tag{Key: "goarch", Value: version.goarch},
+			raven.Tag{Key: "builder", Value: version.builder},
 			raven.Tag{Key: "report_type", Value: reportType},
 		},
 	}
-	if version.Commit != "" {
-		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.Commit})
+	if version.commit != "" {
+		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
 	}
-	for _, tag := range version.Extra {
+	for _, tag := range version.extra {
 		pkt.Tags = append(pkt.Tags, raven.Tag{Key: tag, Value: "1"})
 	}
 	return pkt

cmd/stcrashreceiver/sentry_test.go

@@ -8,12 +8,58 @@ package main
 
 import (
 	"fmt"
-	"os"
+	"io/ioutil"
 	"testing"
 )
 
+func TestParseVersion(t *testing.T) {
+	cases := []struct {
+		longVersion string
+		parsed      version
+	}{
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+			},
+		},
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+				extra:    []string{"foo", "bar"},
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		v, err := parseVersion(tc.longVersion)
+		if err != nil {
+			t.Errorf("%s\nerror: %v\n", tc.longVersion, err)
+			continue
+		}
+		if fmt.Sprint(v) != fmt.Sprint(tc.parsed) {
+			t.Errorf("%s\nA: %v\nE: %v\n", tc.longVersion, v, tc.parsed)
+		}
+	}
+}
+
 func TestParseReport(t *testing.T) {
-	bs, err := os.ReadFile("_testdata/panic.log")
+	bs, err := ioutil.ReadFile("_testdata/panic.log")
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/stcrashreceiver/sourcecodeloader.go

@@ -9,7 +9,7 @@ package main
 import (
 	"bytes"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"net/http"
 	"path/filepath"
 	"strings"
@@ -71,6 +71,7 @@ func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]b
 
 		url := urlPrefix + l.version + filename[idx:]
 		resp, err := l.client.Get(url)
+
 		if err != nil {
 			fmt.Println("Loading source:", err)
 			return nil, 0
@@ -79,7 +80,7 @@ func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]b
 			fmt.Println("Loading source:", resp.Status)
 			return nil, 0
 		}
-		data, err := io.ReadAll(resp.Body)
+		data, err := ioutil.ReadAll(resp.Body)
 		_ = resp.Body.Close()
 		if err != nil {
 			fmt.Println("Loading source:", err.Error())

cmd/stcrashreceiver/stcrashreceiver.go

@@ -0,0 +1,142 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+type crashReceiver struct {
+	dir string
+	dsn string
+}
+
+func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	// The final path component should be a SHA256 hash in hex, so 64 hex
+	// characters. We don't care about case on the request but use lower
+	// case internally.
+	reportID := strings.ToLower(path.Base(req.URL.Path))
+	if len(reportID) != 64 {
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+	for _, c := range reportID {
+		if c >= 'a' && c <= 'f' {
+			continue
+		}
+		if c >= '0' && c <= '9' {
+			continue
+		}
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+
+	// The location of the report on disk, compressed
+	fullPath := filepath.Join(r.dir, r.dirFor(reportID), reportID) + ".gz"
+
+	switch req.Method {
+	case http.MethodGet:
+		r.serveGet(fullPath, w, req)
+	case http.MethodHead:
+		r.serveHead(fullPath, w, req)
+	case http.MethodPut:
+		r.servePut(reportID, fullPath, w, req)
+	default:
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+// serveGet responds to GET requests by serving the uncompressed report.
+func (r *crashReceiver) serveGet(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	fd, err := os.Open(fullPath)
+	if err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+		return
+	}
+
+	defer fd.Close()
+	gr, err := gzip.NewReader(fd)
+	if err != nil {
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+	_, _ = io.Copy(w, gr) // best effort
+}
+
+// serveHead responds to HEAD requests by checking if the named report
+// already exists in the system.
+func (r *crashReceiver) serveHead(fullPath string, w http.ResponseWriter, _ *http.Request) {
+	if _, err := os.Lstat(fullPath); err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+	}
+}
+
+// servePut accepts and stores the given report.
+func (r *crashReceiver) servePut(reportID, fullPath string, w http.ResponseWriter, req *http.Request) {
+	// Ensure the destination directory exists
+	if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
+		log.Println("Creating directory:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Read at most maxRequestSize of report data.
+	log.Println("Receiving report", reportID)
+	lr := io.LimitReader(req.Body, maxRequestSize)
+	bs, err := ioutil.ReadAll(lr)
+	if err != nil {
+		log.Println("Reading report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Compress the report for storage
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	_, _ = gw.Write(bs) // can't fail
+	gw.Close()
+
+	// Create an output file with the compressed report
+	err = ioutil.WriteFile(fullPath, buf.Bytes(), 0644)
+	if err != nil {
+		log.Println("Saving report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Send the report to Sentry
+	if r.dsn != "" {
+		// Remote ID
+		user := userIDFor(req)
+
+		go func() {
+			// There's no need for the client to have to wait for this part.
+			pkt, err := parseCrashReport(reportID, bs)
+			if err != nil {
+				log.Println("Failed to parse crash report:", err)
+				return
+			}
+			if err := sendReport(r.dsn, pkt, user); err != nil {
+				log.Println("Failed to send crash report:", err)
+			}
+		}()
+	}
+}
+
+// 01234567890abcdef... => 01/23
+func (r *crashReceiver) dirFor(base string) string {
+	return filepath.Join(base[0:2], base[2:4])
+}

cmd/stdisco/main.go

@@ -15,10 +15,6 @@ import (
 	"strings"
 	"time"
 
-	"google.golang.org/protobuf/proto"
-
-	"github.com/syncthing/syncthing/internal/gen/discoproto"
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/beacon"
 	"github.com/syncthing/syncthing/lib/discover"
 	"github.com/syncthing/syncthing/lib/protocol"
@@ -78,21 +74,20 @@ func recv(bc beacon.Interface) {
 			continue
 		}
 
-		var ann discoproto.Announce
-		proto.Unmarshal(data[4:], &ann)
+		var ann discover.Announce
+		ann.Unmarshal(data[4:])
 
-		id, _ := protocol.DeviceIDFromBytes(ann.Id)
-		if id == myID {
+		if ann.ID == myID {
 			// This is one of our own fake packets, don't print it.
 			continue
 		}
 
 		// Print announcement details for the first packet from a given
 		// device ID and source address, or if -all was given.
-		key := id.String() + src.String()
+		key := ann.ID.String() + src.String()
 		if all || !seen[key] {
 			log.Printf("Announcement from %v\n", src)
-			log.Printf(" %v at %s\n", id, strings.Join(ann.Addresses, ", "))
+			log.Printf(" %v at %s\n", ann.ID, strings.Join(ann.Addresses, ", "))
 			seen[key] = true
 		}
 	}
@@ -100,11 +95,11 @@ func recv(bc beacon.Interface) {
 
 // sends fake discovery announcements once every second
 func send(bc beacon.Interface) {
-	ann := &discoproto.Announce{
-		Id:        myID[:],
+	ann := discover.Announce{
+		ID:        myID,
 		Addresses: []string{"tcp://fake.example.com:12345"},
 	}
-	bs, _ := proto.Marshal(ann)
+	bs, _ := ann.Marshal()
 
 	for {
 		bc.Send(bs)

cmd/stdiscosrv/amqp.go

@@ -1,261 +0,0 @@
-// Copyright (C) 2024 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"log"
-
-	amqp "github.com/rabbitmq/amqp091-go"
-	"github.com/thejerf/suture/v4"
-	"google.golang.org/protobuf/proto"
-
-	"github.com/syncthing/syncthing/internal/gen/discosrv"
-	"github.com/syncthing/syncthing/internal/protoutil"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-type amqpReplicator struct {
-	suture.Service
-	broker   string
-	sender   *amqpSender
-	receiver *amqpReceiver
-	outbox   chan *discosrv.ReplicationRecord
-}
-
-func newAMQPReplicator(broker, clientID string, db database) *amqpReplicator {
-	svc := suture.New("amqpReplicator", suture.Spec{PassThroughPanics: true})
-
-	sender := &amqpSender{
-		broker:   broker,
-		clientID: clientID,
-		outbox:   make(chan *discosrv.ReplicationRecord, replicationOutboxSize),
-	}
-	svc.Add(sender)
-
-	receiver := &amqpReceiver{
-		broker:   broker,
-		clientID: clientID,
-		db:       db,
-	}
-	svc.Add(receiver)
-
-	return &amqpReplicator{
-		Service:  svc,
-		broker:   broker,
-		sender:   sender,
-		receiver: receiver,
-		outbox:   make(chan *discosrv.ReplicationRecord, replicationOutboxSize),
-	}
-}
-
-func (s *amqpReplicator) send(key *protocol.DeviceID, ps []*discosrv.DatabaseAddress, seen int64) {
-	s.sender.send(key, ps, seen)
-}
-
-type amqpSender struct {
-	broker   string
-	clientID string
-	outbox   chan *discosrv.ReplicationRecord
-}
-
-func (s *amqpSender) Serve(ctx context.Context) error {
-	conn, ch, err := amqpChannel(s.broker)
-	if err != nil {
-		return err
-	}
-	defer ch.Close()
-	defer conn.Close()
-
-	buf := make([]byte, 1024)
-	for {
-		select {
-		case rec := <-s.outbox:
-			size := proto.Size(rec)
-			if len(buf) < size {
-				buf = make([]byte, size)
-			}
-
-			n, err := protoutil.MarshalTo(buf, rec)
-			if err != nil {
-				replicationSendsTotal.WithLabelValues("error").Inc()
-				return fmt.Errorf("replication marshal: %w", err)
-			}
-
-			err = ch.PublishWithContext(ctx,
-				"discovery", // exchange
-				"",          // routing key
-				false,       // mandatory
-				false,       // immediate
-				amqp.Publishing{
-					ContentType: "application/protobuf",
-					Body:        buf[:n],
-					AppId:       s.clientID,
-				})
-			if err != nil {
-				replicationSendsTotal.WithLabelValues("error").Inc()
-				return fmt.Errorf("replication publish: %w", err)
-			}
-
-			replicationSendsTotal.WithLabelValues("success").Inc()
-
-		case <-ctx.Done():
-			return nil
-		}
-	}
-}
-
-func (s *amqpSender) String() string {
-	return fmt.Sprintf("amqpSender(%q)", s.broker)
-}
-
-func (s *amqpSender) send(key *protocol.DeviceID, ps []*discosrv.DatabaseAddress, seen int64) {
-	item := &discosrv.ReplicationRecord{
-		Key:       key[:],
-		Addresses: ps,
-		Seen:      seen,
-	}
-
-	// The send should never block. The inbox is suitably buffered for at
-	// least a few seconds of stalls, which shouldn't happen in practice.
-	select {
-	case s.outbox <- item:
-	default:
-		replicationSendsTotal.WithLabelValues("drop").Inc()
-	}
-}
-
-type amqpReceiver struct {
-	broker   string
-	clientID string
-	db       database
-}
-
-func (s *amqpReceiver) Serve(ctx context.Context) error {
-	conn, ch, err := amqpChannel(s.broker)
-	if err != nil {
-		return err
-	}
-	defer ch.Close()
-	defer conn.Close()
-
-	msgs, err := amqpConsume(ch)
-	if err != nil {
-		return err
-	}
-
-	for {
-		select {
-		case msg, ok := <-msgs:
-			if !ok {
-				return fmt.Errorf("subscription closed: %w", io.EOF)
-			}
-
-			// ignore messages from ourself
-			if msg.AppId == s.clientID {
-				continue
-			}
-
-			var rec discosrv.ReplicationRecord
-			if err := proto.Unmarshal(msg.Body, &rec); err != nil {
-				replicationRecvsTotal.WithLabelValues("error").Inc()
-				return fmt.Errorf("replication unmarshal: %w", err)
-			}
-			id, err := protocol.DeviceIDFromBytes(rec.Key)
-			if err != nil {
-				id, err = protocol.DeviceIDFromString(string(rec.Key))
-			}
-			if err != nil {
-				log.Println("Replication device ID:", err)
-				replicationRecvsTotal.WithLabelValues("error").Inc()
-				continue
-			}
-
-			if err := s.db.merge(&id, rec.Addresses, rec.Seen); err != nil {
-				return fmt.Errorf("replication database merge: %w", err)
-			}
-
-			replicationRecvsTotal.WithLabelValues("success").Inc()
-
-		case <-ctx.Done():
-			return nil
-		}
-	}
-}
-
-func (s *amqpReceiver) String() string {
-	return fmt.Sprintf("amqpReceiver(%q)", s.broker)
-}
-
-func amqpChannel(dst string) (*amqp.Connection, *amqp.Channel, error) {
-	conn, err := amqp.Dial(dst)
-	if err != nil {
-		return nil, nil, fmt.Errorf("AMQP dial: %w", err)
-	}
-
-	ch, err := conn.Channel()
-	if err != nil {
-		return nil, nil, fmt.Errorf("AMQP channel: %w", err)
-	}
-
-	err = ch.ExchangeDeclare(
-		"discovery", // name
-		"fanout",    // type
-		false,       // durable
-		false,       // auto-deleted
-		false,       // internal
-		false,       // no-wait
-		nil,         // arguments
-	)
-	if err != nil {
-		return nil, nil, fmt.Errorf("AMQP declare exchange: %w", err)
-	}
-
-	return conn, ch, nil
-}
-
-func amqpConsume(ch *amqp.Channel) (<-chan amqp.Delivery, error) {
-	q, err := ch.QueueDeclare(
-		"",    // name
-		false, // durable
-		false, // delete when unused
-		true,  // exclusive
-		false, // no-wait
-		nil,   // arguments
-	)
-	if err != nil {
-		return nil, fmt.Errorf("AMQP declare queue: %w", err)
-	}
-
-	err = ch.QueueBind(
-		q.Name,      // queue name
-		"",          // routing key
-		"discovery", // exchange
-		false,
-		nil,
-	)
-	if err != nil {
-		return nil, fmt.Errorf("AMQP bind queue: %w", err)
-	}
-
-	msgs, err := ch.Consume(
-		q.Name, // queue
-		"",     // consumer
-		true,   // auto-ack
-		false,  // exclusive
-		false,  // no-local
-		false,  // no-wait
-		nil,    // args
-	)
-	if err != nil {
-		return nil, fmt.Errorf("AMQP consume: %w", err)
-	}
-
-	return msgs, nil
-}

cmd/stdiscosrv/apisrv.go

@@ -8,29 +8,23 @@ package main
 
 import (
 	"bytes"
-	"compress/gzip"
 	"context"
 	"crypto/tls"
-	"encoding/base64"
 	"encoding/json"
 	"encoding/pem"
-	"errors"
 	"fmt"
-	io "io"
 	"log"
 	"math/rand"
 	"net"
 	"net/http"
 	"net/url"
-	"slices"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/syncthing/syncthing/internal/gen/discosrv"
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/stringutil"
 )
 
 // announcement is the format received from and sent to clients
@@ -40,20 +34,15 @@ type announcement struct {
 }
 
 type apiSrv struct {
-	addr           string
-	cert           tls.Certificate
-	db             database
-	listener       net.Listener
-	repl           replicator // optional
-	useHTTP        bool
-	compression    bool
-	gzipWriters    sync.Pool
-	seenTracker    *retryAfterTracker
-	notSeenTracker *retryAfterTracker
-}
+	addr     string
+	cert     tls.Certificate
+	db       database
+	listener net.Listener
+	repl     replicator // optional
+	useHTTP  bool
 
-type replicator interface {
-	send(key *protocol.DeviceID, addrs []*discosrv.DatabaseAddress, seen int64)
+	mapsMut sync.Mutex
+	misses  map[string]int32
 }
 
 type requestID int64
@@ -66,26 +55,14 @@ type contextKey int
 
 const idKey contextKey = iota
 
-func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP, compression bool) *apiSrv {
+func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP bool) *apiSrv {
 	return &apiSrv{
-		addr:        addr,
-		cert:        cert,
-		db:          db,
-		repl:        repl,
-		useHTTP:     useHTTP,
-		compression: compression,
-		seenTracker: &retryAfterTracker{
-			name:         "seenTracker",
-			bucketStarts: time.Now(),
-			desiredRate:  250,
-			currentDelay: notFoundRetryUnknownMinSeconds,
-		},
-		notSeenTracker: &retryAfterTracker{
-			name:         "notSeenTracker",
-			bucketStarts: time.Now(),
-			desiredRate:  250,
-			currentDelay: notFoundRetryUnknownMaxSeconds / 2,
-		},
+		addr:    addr,
+		cert:    cert,
+		db:      db,
+		repl:    repl,
+		useHTTP: useHTTP,
+		misses:  make(map[string]int32),
 	}
 }
 
@@ -99,10 +76,18 @@ func (s *apiSrv) Serve(ctx context.Context) error {
 		s.listener = listener
 	} else {
 		tlsCfg := &tls.Config{
-			Certificates: []tls.Certificate{s.cert},
-			ClientAuth:   tls.RequestClientCert,
-			MinVersion:   tls.VersionTLS12,
-			NextProtos:   []string{"h2", "http/1.1"},
+			Certificates:           []tls.Certificate{s.cert},
+			ClientAuth:             tls.RequestClientCert,
+			SessionTicketsDisabled: true,
+			MinVersion:             tls.VersionTLS12,
+			CipherSuites: []uint16{
+				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+			},
 		}
 
 		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
@@ -121,14 +106,6 @@ func (s *apiSrv) Serve(ctx context.Context) error {
 		WriteTimeout:   httpWriteTimeout,
 		MaxHeaderBytes: httpMaxHeaderBytes,
 	}
-	if !debug {
-		srv.ErrorLog = log.New(io.Discard, "", 0)
-	}
-
-	go func() {
-		<-ctx.Done()
-		srv.Shutdown(context.Background())
-	}()
 
 	err := srv.Serve(s.listener)
 	if err != nil {
@@ -137,6 +114,8 @@ func (s *apiSrv) Serve(ctx context.Context) error {
 	return err
 }
 
+var topCtx = context.Background()
+
 func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 	t0 := time.Now()
 
@@ -149,10 +128,10 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 	}()
 
 	reqID := requestID(rand.Int63())
-	req = req.WithContext(context.WithValue(req.Context(), idKey, reqID))
+	ctx := context.WithValue(topCtx, idKey, reqID)
 
 	if debug {
-		log.Println(reqID, req.Method, req.URL, req.Proto)
+		log.Println(reqID, req.Method, req.URL)
 	}
 
 	remoteAddr := &net.TCPAddr{
@@ -161,12 +140,7 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 	}
 
 	if s.useHTTP {
-		// X-Forwarded-For can have multiple client IPs; split using the comma separator
-		forwardIP, _, _ := strings.Cut(req.Header.Get("X-Forwarded-For"), ",")
-
-		// net.ParseIP will return nil if leading/trailing whitespace exists; use strings.TrimSpace()
-		remoteAddr.IP = net.ParseIP(strings.TrimSpace(forwardIP))
-
+		remoteAddr.IP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
 		if parsedPort, err := strconv.ParseInt(req.Header.Get("X-Client-Port"), 10, 0); err == nil {
 			remoteAddr.Port = int(parsedPort)
 		}
@@ -183,22 +157,22 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 	}
 
 	switch req.Method {
-	case http.MethodGet:
-		s.handleGET(lw, req)
-	case http.MethodPost:
-		s.handlePOST(remoteAddr, lw, req)
+	case "GET":
+		s.handleGET(ctx, lw, req)
+	case "POST":
+		s.handlePOST(ctx, remoteAddr, lw, req)
 	default:
 		http.Error(lw, "Method Not Allowed", http.StatusMethodNotAllowed)
 	}
 }
 
-func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
-	reqID := req.Context().Value(idKey).(requestID)
+func (s *apiSrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
+	reqID := ctx.Value(idKey).(requestID)
 
 	deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
 	if err != nil {
 		if debug {
-			log.Println(reqID, "bad device param:", err)
+			log.Println(reqID, "bad device param")
 		}
 		lookupRequestsTotal.WithLabelValues("bad_request").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
@@ -206,7 +180,8 @@ func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	rec, err := s.db.get(&deviceID)
+	key := deviceID.String()
+	rec, err := s.db.get(key)
 	if err != nil {
 		// some sort of internal error
 		lookupRequestsTotal.WithLabelValues("internal_error").Inc()
@@ -216,51 +191,48 @@ func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
 	}
 
 	if len(rec.Addresses) == 0 {
-		var afterS int
-		if rec.Seen == 0 {
-			afterS = s.notSeenTracker.retryAfterS()
-			lookupRequestsTotal.WithLabelValues("not_found_ever").Inc()
+		lookupRequestsTotal.WithLabelValues("not_found").Inc()
+
+		s.mapsMut.Lock()
+		misses := s.misses[key]
+		if misses < rec.Misses {
+			misses = rec.Misses + 1
 		} else {
-			afterS = s.seenTracker.retryAfterS()
-			lookupRequestsTotal.WithLabelValues("not_found_recent").Inc()
+			misses++
 		}
-		w.Header().Set("Retry-After", strconv.Itoa(afterS))
+		s.misses[key] = misses
+		s.mapsMut.Unlock()
+
+		if misses%notFoundMissesWriteInterval == 0 {
+			rec.Misses = misses
+			rec.Missed = time.Now().UnixNano()
+			rec.Addresses = nil
+			// rec.Seen retained from get
+			s.db.put(key, rec)
+		}
+
+		w.Header().Set("Retry-After", notFoundRetryAfterString(int(misses)))
 		http.Error(w, "Not Found", http.StatusNotFound)
 		return
 	}
 
 	lookupRequestsTotal.WithLabelValues("success").Inc()
 
-	w.Header().Set("Content-Type", "application/json")
-	var bw io.Writer = w
-
-	// Use compression if the client asks for it
-	if s.compression && strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
-		gw, ok := s.gzipWriters.Get().(*gzip.Writer)
-		if ok {
-			gw.Reset(w)
-		} else {
-			gw = gzip.NewWriter(w)
-		}
-		w.Header().Set("Content-Encoding", "gzip")
-		defer gw.Close()
-		defer s.gzipWriters.Put(gw)
-		bw = gw
-	}
-
-	json.NewEncoder(bw).Encode(announcement{
-		Seen:      time.Unix(0, rec.Seen).Truncate(time.Second),
+	bs, _ := json.Marshal(announcement{
+		Seen:      time.Unix(0, rec.Seen),
 		Addresses: addressStrs(rec.Addresses),
 	})
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bs)
 }
 
-func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req *http.Request) {
-	reqID := req.Context().Value(idKey).(requestID)
+func (s *apiSrv) handlePOST(ctx context.Context, remoteAddr *net.TCPAddr, w http.ResponseWriter, req *http.Request) {
+	reqID := ctx.Value(idKey).(requestID)
 
-	rawCert, err := certificateBytes(req)
-	if err != nil {
+	rawCert := certificateBytes(req)
+	if rawCert == nil {
 		if debug {
-			log.Println(reqID, "no certificates:", err)
+			log.Println(reqID, "no certificates")
 		}
 		announceRequestsTotal.WithLabelValues("no_certificate").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
@@ -283,9 +255,6 @@ func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req
 
 	addresses := fixupAddresses(remoteAddr, ann.Addresses)
 	if len(addresses) == 0 {
-		if debug {
-			log.Println(reqID, "no addresses")
-		}
 		announceRequestsTotal.WithLabelValues("bad_request").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
 		http.Error(w, "Bad Request", http.StatusBadRequest)
@@ -293,9 +262,6 @@ func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req
 	}
 
 	if err := s.handleAnnounce(deviceID, addresses); err != nil {
-		if debug {
-			log.Println(reqID, "handle:", err)
-		}
 		announceRequestsTotal.WithLabelValues("internal_error").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
@@ -306,9 +272,6 @@ func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req
 
 	w.Header().Set("Reannounce-After", reannounceAfterString())
 	w.WriteHeader(http.StatusNoContent)
-	if debug {
-		log.Println(reqID, "announced", deviceID, addresses)
-	}
 }
 
 func (s *apiSrv) Stop() {
@@ -316,36 +279,34 @@ func (s *apiSrv) Stop() {
 }
 
 func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string) error {
+	key := deviceID.String()
 	now := time.Now()
 	expire := now.Add(addressExpiryTime).UnixNano()
 
+	dbAddrs := make([]DatabaseAddress, len(addresses))
+	for i := range addresses {
+		dbAddrs[i].Address = addresses[i]
+		dbAddrs[i].Expires = expire
+	}
+
 	// The address slice must always be sorted for database merges to work
 	// properly.
-	slices.Sort(addresses)
-	addresses = slices.Compact(addresses)
-
-	dbAddrs := make([]*discosrv.DatabaseAddress, len(addresses))
-	for i := range addresses {
-		dbAddrs[i] = &discosrv.DatabaseAddress{
-			Address: addresses[i],
-			Expires: expire,
-		}
-	}
+	sort.Sort(databaseAddressOrder(dbAddrs))
 
 	seen := now.UnixNano()
 	if s.repl != nil {
-		s.repl.send(&deviceID, dbAddrs, seen)
+		s.repl.send(key, dbAddrs, seen)
 	}
-	return s.db.merge(&deviceID, dbAddrs, seen)
+	return s.db.merge(key, dbAddrs, seen)
 }
 
-func handlePing(w http.ResponseWriter, _ *http.Request) {
-	w.WriteHeader(http.StatusNoContent)
+func handlePing(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(204)
 }
 
-func certificateBytes(req *http.Request) ([]byte, error) {
+func certificateBytes(req *http.Request) []byte {
 	if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
-		return req.TLS.PeerCertificates[0].Raw, nil
+		return req.TLS.PeerCertificates[0].Raw
 	}
 
 	var bs []byte
@@ -358,7 +319,7 @@ func certificateBytes(req *http.Request) ([]byte, error) {
 			hdr, err := url.QueryUnescape(hdr)
 			if err != nil {
 				// Decoding failed
-				return nil, err
+				return nil
 			}
 
 			bs = []byte(hdr)
@@ -377,66 +338,37 @@ func certificateBytes(req *http.Request) ([]byte, error) {
 				}
 			}
 		}
-	} else if hdr := req.Header.Get("X-Tls-Client-Cert-Der-Base64"); hdr != "" {
-		// Caddy {tls_client_certificate_der_base64}
-		hdr, err := base64.StdEncoding.DecodeString(hdr)
+	} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
+		// Traefik 2 passtlsclientcert
+		// The certificate is in PEM format with url encoding but without newlines
+		// and start/end statements. We need to decode, reinstate the newlines every 64
+		// character and add statements for the PEM decoder
+		hdr, err := url.QueryUnescape(hdr)
 		if err != nil {
 			// Decoding failed
-			return nil, err
+			return nil
 		}
 
-		bs = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: hdr})
-	} else if cert := req.Header.Get("X-Forwarded-Tls-Client-Cert"); cert != "" {
-		// Traefik 2 passtlsclientcert
-		//
-		// The certificate is in PEM format, maybe with URL encoding
-		// (depends on Traefik version) but without newlines and start/end
-		// statements. We need to decode, reinstate the newlines every 64
-		// character and add statements for the PEM decoder
-
-		if strings.Contains(cert, "%") {
-			if unesc, err := url.QueryUnescape(cert); err == nil {
-				cert = unesc
-			}
+		for i := 64; i < len(hdr); i += 65 {
+			hdr = hdr[:i] + "\n" + hdr[i:]
 		}
 
-		const (
-			header = "-----BEGIN CERTIFICATE-----"
-			footer = "-----END CERTIFICATE-----"
-		)
-
-		var b bytes.Buffer
-		b.Grow(len(header) + 1 + len(cert) + len(cert)/64 + 1 + len(footer) + 1)
-
-		b.WriteString(header)
-		b.WriteByte('\n')
-
-		for i := 0; i < len(cert); i += 64 {
-			end := i + 64
-			if end > len(cert) {
-				end = len(cert)
-			}
-			b.WriteString(cert[i:end])
-			b.WriteByte('\n')
-		}
-
-		b.WriteString(footer)
-		b.WriteByte('\n')
-
-		bs = b.Bytes()
+		hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
+		hdr = hdr + "\n-----END CERTIFICATE-----\n"
+		bs = []byte(hdr)
 	}
 
 	if bs == nil {
-		return nil, errors.New("empty certificate header")
+		return nil
 	}
 
 	block, _ := pem.Decode(bs)
 	if block == nil {
 		// Decoding failed
-		return nil, errors.New("certificate decode result is empty")
+		return nil
 	}
 
-	return block.Bytes, nil
+	return block.Bytes
 }
 
 // fixupAddresses checks the list of addresses, removing invalid ones and
@@ -461,13 +393,13 @@ func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
 			continue
 		}
 
-		if host == "" || ip.IsUnspecified() {
-			if remote != nil {
+		if remote != nil {
+			if host == "" || ip.IsUnspecified() {
 				// Replace the unspecified IP with the request source.
 
 				// ... unless the request source is the loopback address or
 				// multicast/unspecified (can't happen, really).
-				if remote.IP == nil || remote.IP.IsLoopback() || remote.IP.IsMulticast() || remote.IP.IsUnspecified() {
+				if remote.IP.IsLoopback() || remote.IP.IsMulticast() || remote.IP.IsUnspecified() {
 					continue
 				}
 
@@ -483,21 +415,11 @@ func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
 				}
 
 				host = remote.IP.String()
-
-			} else {
-				// remote is nil, unable to determine host IP
-				continue
 			}
-		}
 
-		// If zero port was specified, use remote port.
-		if port == "0" {
-			if remote != nil && remote.Port > 0 {
-				// use remote port
-				port = strconv.Itoa(remote.Port)
-			} else {
-				// unable to determine remote port
-				continue
+			// If zero port was specified, use remote port.
+			if port == "0" && remote.Port > 0 {
+				port = fmt.Sprintf("%d", remote.Port)
 			}
 		}
 
@@ -505,9 +427,6 @@ func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
 		fixed = append(fixed, uri.String())
 	}
 
-	// Remove duplicate addresses
-	fixed = stringutil.UniqueTrimmedStrings(fixed)
-
 	return fixed
 }
 
@@ -525,7 +444,7 @@ func (lrw *loggingResponseWriter) WriteHeader(code int) {
 	lrw.ResponseWriter.WriteHeader(code)
 }
 
-func addressStrs(dbAddrs []*discosrv.DatabaseAddress) []string {
+func addressStrs(dbAddrs []DatabaseAddress) []string {
 	res := make([]string, len(dbAddrs))
 	for i, a := range dbAddrs {
 		res[i] = a.Address
@@ -537,44 +456,15 @@ func errorRetryAfterString() string {
 	return strconv.Itoa(errorRetryAfterSeconds + rand.Intn(errorRetryFuzzSeconds))
 }
 
+func notFoundRetryAfterString(misses int) string {
+	retryAfterS := notFoundRetryMinSeconds + notFoundRetryIncSeconds*misses
+	if retryAfterS > notFoundRetryMaxSeconds {
+		retryAfterS = notFoundRetryMaxSeconds
+	}
+	retryAfterS += rand.Intn(notFoundRetryFuzzSeconds)
+	return strconv.Itoa(retryAfterS)
+}
+
 func reannounceAfterString() string {
 	return strconv.Itoa(reannounceAfterSeconds + rand.Intn(reannounzeFuzzSeconds))
 }
-
-type retryAfterTracker struct {
-	name        string
-	desiredRate float64 // requests per second
-
-	mut          sync.Mutex
-	lastCount    int       // requests in the last bucket
-	curCount     int       // requests in the current bucket
-	bucketStarts time.Time // start of the current bucket
-	currentDelay int       // current delay in seconds
-}
-
-func (t *retryAfterTracker) retryAfterS() int {
-	now := time.Now()
-	t.mut.Lock()
-	if durS := now.Sub(t.bucketStarts).Seconds(); durS > float64(t.currentDelay) {
-		t.bucketStarts = now
-		t.lastCount = t.curCount
-		lastRate := float64(t.lastCount) / durS
-
-		switch {
-		case t.currentDelay > notFoundRetryUnknownMinSeconds &&
-			lastRate < 0.75*t.desiredRate:
-			t.currentDelay = max(8*t.currentDelay/10, notFoundRetryUnknownMinSeconds)
-		case t.currentDelay < notFoundRetryUnknownMaxSeconds &&
-			lastRate > 1.25*t.desiredRate:
-			t.currentDelay = min(3*t.currentDelay/2, notFoundRetryUnknownMaxSeconds)
-		}
-
-		t.curCount = 0
-	}
-	if t.curCount == 0 {
-		retryAfterLevel.WithLabelValues(t.name).Set(float64(t.currentDelay))
-	}
-	t.curCount++
-	t.mut.Unlock()
-	return t.currentDelay + rand.Intn(t.currentDelay/4)
-}

cmd/stdiscosrv/apisrv_test.go

@@ -7,20 +7,9 @@
 package main
 
 import (
-	"context"
-	"crypto/tls"
 	"fmt"
-	"io"
 	"net"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"regexp"
-	"strings"
 	"testing"
-
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/tlsutil"
 )
 
 func TestFixupAddresses(t *testing.T) {
@@ -80,14 +69,6 @@ func TestFixupAddresses(t *testing.T) {
 			remote: addr("123.123.123.123", 9000),
 			in:     []string{"tcp://44.44.44.44:0"},
 			out:    []string{"tcp://44.44.44.44:9000"},
-		}, { // remote ip nil
-			remote: addr("", 9000),
-			in:     []string{"tcp://:22000", "tcp://44.44.44.44:9000"},
-			out:    []string{"tcp://44.44.44.44:9000"},
-		}, { // remote port 0
-			remote: addr("123.123.123.123", 0),
-			in:     []string{"tcp://:22000", "tcp://44.44.44.44"},
-			out:    []string{"tcp://123.123.123.123:22000"},
 		},
 	}
 
@@ -105,79 +86,3 @@ func addr(host string, port int) *net.TCPAddr {
 		Port: port,
 	}
 }
-
-func BenchmarkAPIRequests(b *testing.B) {
-	db := newInMemoryStore(b.TempDir(), 0, nil)
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go db.Serve(ctx)
-	api := newAPISrv("127.0.0.1:0", tls.Certificate{}, db, nil, true, true)
-	srv := httptest.NewServer(http.HandlerFunc(api.handler))
-
-	kf := b.TempDir() + "/cert"
-	crt, err := tlsutil.NewCertificate(kf+".crt", kf+".key", "localhost", 7)
-	if err != nil {
-		b.Fatal(err)
-	}
-	certBs, err := os.ReadFile(kf + ".crt")
-	if err != nil {
-		b.Fatal(err)
-	}
-	certBs = regexp.MustCompile(`---[^\n]+---\n`).ReplaceAll(certBs, nil)
-	certString := string(strings.ReplaceAll(string(certBs), "\n", " "))
-
-	devID := protocol.NewDeviceID(crt.Certificate[0])
-	devIDString := devID.String()
-
-	b.Run("Announce", func(b *testing.B) {
-		b.ReportAllocs()
-		url := srv.URL + "/v2/?device=" + devIDString
-		for i := 0; i < b.N; i++ {
-			req, _ := http.NewRequest(http.MethodPost, url, strings.NewReader(`{"addresses":["tcp://10.10.10.10:42000"]}`))
-			req.Header.Set("X-Forwarded-Tls-Client-Cert", certString)
-			resp, err := http.DefaultClient.Do(req)
-			if err != nil {
-				b.Fatal(err)
-			}
-			resp.Body.Close()
-			if resp.StatusCode != http.StatusNoContent {
-				b.Fatalf("unexpected status %s", resp.Status)
-			}
-		}
-	})
-
-	b.Run("Lookup", func(b *testing.B) {
-		b.ReportAllocs()
-		url := srv.URL + "/v2/?device=" + devIDString
-		for i := 0; i < b.N; i++ {
-			req, _ := http.NewRequest(http.MethodGet, url, nil)
-			resp, err := http.DefaultClient.Do(req)
-			if err != nil {
-				b.Fatal(err)
-			}
-			io.Copy(io.Discard, resp.Body)
-			resp.Body.Close()
-			if resp.StatusCode != http.StatusOK {
-				b.Fatalf("unexpected status %s", resp.Status)
-			}
-		}
-	})
-
-	b.Run("LookupNoCompression", func(b *testing.B) {
-		b.ReportAllocs()
-		url := srv.URL + "/v2/?device=" + devIDString
-		for i := 0; i < b.N; i++ {
-			req, _ := http.NewRequest(http.MethodGet, url, nil)
-			req.Header.Set("Accept-Encoding", "identity") // disable compression
-			resp, err := http.DefaultClient.Do(req)
-			if err != nil {
-				b.Fatal(err)
-			}
-			io.Copy(io.Discard, resp.Body)
-			resp.Body.Close()
-			if resp.StatusCode != http.StatusOK {
-				b.Fatalf("unexpected status %s", resp.Status)
-			}
-		}
-	})
-}

cmd/stdiscosrv/database.go

@@ -4,31 +4,19 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
+//go:generate go run ../../proto/scripts/protofmt.go database.proto
+//go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
+
 package main
 
 import (
-	"bufio"
-	"cmp"
 	"context"
-	"encoding/binary"
-	"errors"
-	"io"
 	"log"
-	"os"
-	"path"
-	"runtime"
-	"slices"
-	"strings"
+	"sort"
 	"time"
 
-	"github.com/puzpuzpuz/xsync/v3"
-	"google.golang.org/protobuf/proto"
-
-	"github.com/syncthing/syncthing/internal/gen/discosrv"
-	"github.com/syncthing/syncthing/internal/protoutil"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/rand"
-	"github.com/syncthing/syncthing/lib/s3"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 type clock interface {
@@ -42,401 +30,347 @@ func (defaultClock) Now() time.Time {
 }
 
 type database interface {
-	put(key *protocol.DeviceID, rec *discosrv.DatabaseRecord) error
-	merge(key *protocol.DeviceID, addrs []*discosrv.DatabaseAddress, seen int64) error
-	get(key *protocol.DeviceID) (*discosrv.DatabaseRecord, error)
+	put(key string, rec DatabaseRecord) error
+	merge(key string, addrs []DatabaseAddress, seen int64) error
+	get(key string) (DatabaseRecord, error)
 }
 
-type inMemoryStore struct {
-	m             *xsync.MapOf[protocol.DeviceID, *discosrv.DatabaseRecord]
-	dir           string
-	flushInterval time.Duration
-	s3            *s3.Session
-	objKey        string
-	clock         clock
+type levelDBStore struct {
+	db         *leveldb.DB
+	inbox      chan func()
+	clock      clock
+	marshalBuf []byte
 }
 
-func newInMemoryStore(dir string, flushInterval time.Duration, s3sess *s3.Session) *inMemoryStore {
-	hn, err := os.Hostname()
+func newLevelDBStore(dir string) (*levelDBStore, error) {
+	db, err := leveldb.OpenFile(dir, levelDBOptions)
 	if err != nil {
-		hn = rand.String(8)
+		return nil, err
 	}
-	s := &inMemoryStore{
-		m:             xsync.NewMapOf[protocol.DeviceID, *discosrv.DatabaseRecord](),
-		dir:           dir,
-		flushInterval: flushInterval,
-		s3:            s3sess,
-		objKey:        hn + ".db",
-		clock:         defaultClock{},
-	}
-	nr, err := s.read()
-	if os.IsNotExist(err) && s3sess != nil {
-		// Try to read from AWS
-		latestKey, cerr := s3sess.LatestKey()
-		if cerr != nil {
-			log.Println("Error reading database from S3:", err)
-			return s
-		}
-		fd, cerr := os.Create(path.Join(s.dir, "records.db"))
-		if cerr != nil {
-			log.Println("Error creating database file:", err)
-			return s
-		}
-		if cerr := s3sess.Download(fd, latestKey); cerr != nil {
-			log.Printf("Error reading database from S3: %v", err)
-		}
-		_ = fd.Close()
-		nr, err = s.read()
+	return &levelDBStore{
+		db:    db,
+		inbox: make(chan func(), 16),
+		clock: defaultClock{},
+	}, nil
+}
+
+func (s *levelDBStore) put(key string, rec DatabaseRecord) error {
+	t0 := time.Now()
+	defer func() {
+		databaseOperationSeconds.WithLabelValues(dbOpPut).Observe(time.Since(t0).Seconds())
+	}()
+
+	rc := make(chan error)
+
+	s.inbox <- func() {
+		size := rec.Size()
+		if len(s.marshalBuf) < size {
+			s.marshalBuf = make([]byte, size)
+		}
+		n, _ := rec.MarshalTo(s.marshalBuf)
+		rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
 	}
+
+	err := <-rc
 	if err != nil {
-		log.Println("Error reading database:", err)
+		databaseOperations.WithLabelValues(dbOpPut, dbResError).Inc()
+	} else {
+		databaseOperations.WithLabelValues(dbOpPut, dbResSuccess).Inc()
 	}
-	log.Printf("Read %d records from database", nr)
-	s.expireAndCalculateStatistics()
-	return s
+
+	return err
 }
 
-func (s *inMemoryStore) put(key *protocol.DeviceID, rec *discosrv.DatabaseRecord) error {
+func (s *levelDBStore) merge(key string, addrs []DatabaseAddress, seen int64) error {
 	t0 := time.Now()
-	s.m.Store(*key, rec)
-	databaseOperations.WithLabelValues(dbOpPut, dbResSuccess).Inc()
-	databaseOperationSeconds.WithLabelValues(dbOpPut).Observe(time.Since(t0).Seconds())
-	return nil
-}
+	defer func() {
+		databaseOperationSeconds.WithLabelValues(dbOpMerge).Observe(time.Since(t0).Seconds())
+	}()
 
-func (s *inMemoryStore) merge(key *protocol.DeviceID, addrs []*discosrv.DatabaseAddress, seen int64) error {
-	t0 := time.Now()
-
-	newRec := &discosrv.DatabaseRecord{
+	rc := make(chan error)
+	newRec := DatabaseRecord{
 		Addresses: addrs,
 		Seen:      seen,
 	}
 
-	if oldRec, ok := s.m.Load(*key); ok {
-		newRec = merge(oldRec, newRec)
+	s.inbox <- func() {
+		// grab the existing record
+		oldRec, err := s.get(key)
+		if err != nil {
+			// "not found" is not an error from get, so this is serious
+			// stuff only
+			rc <- err
+			return
+		}
+		newRec = merge(newRec, oldRec)
+
+		// We replicate s.put() functionality here ourselves instead of
+		// calling it because we want to serialize our get above together
+		// with the put in the same function.
+		size := newRec.Size()
+		if len(s.marshalBuf) < size {
+			s.marshalBuf = make([]byte, size)
+		}
+		n, _ := newRec.MarshalTo(s.marshalBuf)
+		rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
 	}
-	s.m.Store(*key, newRec)
 
-	databaseOperations.WithLabelValues(dbOpMerge, dbResSuccess).Inc()
-	databaseOperationSeconds.WithLabelValues(dbOpMerge).Observe(time.Since(t0).Seconds())
+	err := <-rc
+	if err != nil {
+		databaseOperations.WithLabelValues(dbOpMerge, dbResError).Inc()
+	} else {
+		databaseOperations.WithLabelValues(dbOpMerge, dbResSuccess).Inc()
+	}
 
-	return nil
+	return err
 }
 
-func (s *inMemoryStore) get(key *protocol.DeviceID) (*discosrv.DatabaseRecord, error) {
+func (s *levelDBStore) get(key string) (DatabaseRecord, error) {
 	t0 := time.Now()
 	defer func() {
 		databaseOperationSeconds.WithLabelValues(dbOpGet).Observe(time.Since(t0).Seconds())
 	}()
 
-	rec, ok := s.m.Load(*key)
-	if !ok {
+	keyBs := []byte(key)
+	val, err := s.db.Get(keyBs, nil)
+	if err == leveldb.ErrNotFound {
 		databaseOperations.WithLabelValues(dbOpGet, dbResNotFound).Inc()
-		return &discosrv.DatabaseRecord{}, nil
+		return DatabaseRecord{}, nil
+	}
+	if err != nil {
+		databaseOperations.WithLabelValues(dbOpGet, dbResError).Inc()
+		return DatabaseRecord{}, err
 	}
 
-	rec.Addresses = expire(rec.Addresses, s.clock.Now())
+	var rec DatabaseRecord
+
+	if err := rec.Unmarshal(val); err != nil {
+		databaseOperations.WithLabelValues(dbOpGet, dbResUnmarshalError).Inc()
+		return DatabaseRecord{}, nil
+	}
+
+	rec.Addresses = expire(rec.Addresses, s.clock.Now().UnixNano())
 	databaseOperations.WithLabelValues(dbOpGet, dbResSuccess).Inc()
 	return rec, nil
 }
 
-func (s *inMemoryStore) Serve(ctx context.Context) error {
-	if s.flushInterval <= 0 {
-		<-ctx.Done()
-		return nil
-	}
-
-	t := time.NewTimer(s.flushInterval)
+func (s *levelDBStore) Serve(ctx context.Context) error {
+	t := time.NewTimer(0)
 	defer t.Stop()
+	defer s.db.Close()
+
+	// Start the statistics serve routine. It will exit with us when
+	// statisticsTrigger is closed.
+	statisticsTrigger := make(chan struct{})
+	statisticsDone := make(chan struct{})
+	go s.statisticsServe(statisticsTrigger, statisticsDone)
 
 loop:
 	for {
 		select {
+		case fn := <-s.inbox:
+			// Run function in serialized order.
+			fn()
+
 		case <-t.C:
-			log.Println("Calculating statistics")
-			s.expireAndCalculateStatistics()
-			log.Println("Flushing database")
-			if err := s.write(); err != nil {
-				log.Println("Error writing database:", err)
-			}
-			log.Println("Finished flushing database")
-			t.Reset(s.flushInterval)
+			// Trigger the statistics routine to do its thing in the
+			// background.
+			statisticsTrigger <- struct{}{}
+
+		case <-statisticsDone:
+			// The statistics routine is done with one iteratation, schedule
+			// the next.
+			t.Reset(databaseStatisticsInterval)
 
 		case <-ctx.Done():
 			// We're done.
+			close(statisticsTrigger)
 			break loop
 		}
 	}
 
-	return s.write()
-}
-
-func (s *inMemoryStore) expireAndCalculateStatistics() {
-	now := s.clock.Now()
-	cutoff24h := now.Add(-24 * time.Hour).UnixNano()
-	cutoff1w := now.Add(-7 * 24 * time.Hour).UnixNano()
-	current, currentIPv4, currentIPv6, currentIPv6GUA, last24h, last1w := 0, 0, 0, 0, 0, 0
-
-	n := 0
-	s.m.Range(func(key protocol.DeviceID, rec *discosrv.DatabaseRecord) bool {
-		if n%1000 == 0 {
-			runtime.Gosched()
-		}
-		n++
-
-		addresses := expire(rec.Addresses, now)
-		if len(addresses) == 0 {
-			rec.Addresses = nil
-			s.m.Store(key, rec)
-		} else if len(addresses) != len(rec.Addresses) {
-			rec.Addresses = addresses
-			s.m.Store(key, rec)
-		}
-
-		switch {
-		case len(rec.Addresses) > 0:
-			current++
-			seenIPv4, seenIPv6, seenIPv6GUA := false, false, false
-			for _, addr := range rec.Addresses {
-				// We do fast and loose matching on strings here instead of
-				// parsing the address and the IP and doing "proper" checks,
-				// to keep things fast and generate less garbage.
-				if strings.Contains(addr.Address, "[") {
-					seenIPv6 = true
-					if strings.Contains(addr.Address, "[2") {
-						seenIPv6GUA = true
-					}
-				} else {
-					seenIPv4 = true
-				}
-				if seenIPv4 && seenIPv6 && seenIPv6GUA {
-					break
-				}
-			}
-			if seenIPv4 {
-				currentIPv4++
-			}
-			if seenIPv6 {
-				currentIPv6++
-			}
-			if seenIPv6GUA {
-				currentIPv6GUA++
-			}
-		case rec.Seen > cutoff24h:
-			last24h++
-		case rec.Seen > cutoff1w:
-			last1w++
-		default:
-			// drop the record if it's older than a week
-			s.m.Delete(key)
-		}
-		return true
-	})
-
-	databaseKeys.WithLabelValues("current").Set(float64(current))
-	databaseKeys.WithLabelValues("currentIPv4").Set(float64(currentIPv4))
-	databaseKeys.WithLabelValues("currentIPv6").Set(float64(currentIPv6))
-	databaseKeys.WithLabelValues("currentIPv6GUA").Set(float64(currentIPv6GUA))
-	databaseKeys.WithLabelValues("last24h").Set(float64(last24h))
-	databaseKeys.WithLabelValues("last1w").Set(float64(last1w))
-	databaseStatisticsSeconds.Set(time.Since(now).Seconds())
-}
-
-func (s *inMemoryStore) write() (err error) {
-	t0 := time.Now()
-	defer func() {
-		if err == nil {
-			databaseWriteSeconds.Set(time.Since(t0).Seconds())
-			databaseLastWritten.Set(float64(t0.Unix()))
-		}
-	}()
-
-	dbf := path.Join(s.dir, "records.db")
-	fd, err := os.Create(dbf + ".tmp")
-	if err != nil {
-		return err
-	}
-	bw := bufio.NewWriter(fd)
-
-	var buf []byte
-	var rangeErr error
-	now := s.clock.Now()
-	cutoff1w := now.Add(-7 * 24 * time.Hour).UnixNano()
-	n := 0
-	s.m.Range(func(key protocol.DeviceID, value *discosrv.DatabaseRecord) bool {
-		if n%1000 == 0 {
-			runtime.Gosched()
-		}
-		n++
-
-		if value.Seen < cutoff1w {
-			// drop the record if it's older than a week
-			return true
-		}
-		rec := &discosrv.ReplicationRecord{
-			Key:       key[:],
-			Addresses: value.Addresses,
-			Seen:      value.Seen,
-		}
-		s := proto.Size(rec)
-		if s+4 > len(buf) {
-			buf = make([]byte, s+4)
-		}
-		n, err := protoutil.MarshalTo(buf[4:], rec)
-		if err != nil {
-			rangeErr = err
-			return false
-		}
-		binary.BigEndian.PutUint32(buf, uint32(n))
-		if _, err := bw.Write(buf[:n+4]); err != nil {
-			rangeErr = err
-			return false
-		}
-		return true
-	})
-	if rangeErr != nil {
-		_ = fd.Close()
-		return rangeErr
-	}
-
-	if err := bw.Flush(); err != nil {
-		_ = fd.Close
-		return err
-	}
-	if err := fd.Close(); err != nil {
-		return err
-	}
-	if err := os.Rename(dbf+".tmp", dbf); err != nil {
-		return err
-	}
-
-	// Upload to S3
-	if s.s3 != nil {
-		fd, err = os.Open(dbf)
-		if err != nil {
-			log.Printf("Error uploading database to S3: %v", err)
-			return nil
-		}
-		defer fd.Close()
-		if err := s.s3.Upload(fd, s.objKey); err != nil {
-			log.Printf("Error uploading database to S3: %v", err)
-		}
-		log.Println("Finished uploading database")
-	}
+	// Also wait for statisticsServe to return
+	<-statisticsDone
 
 	return nil
 }
 
-func (s *inMemoryStore) read() (int, error) {
-	fd, err := os.Open(path.Join(s.dir, "records.db"))
-	if err != nil {
-		return 0, err
-	}
-	defer fd.Close()
+func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
+	defer close(done)
 
-	br := bufio.NewReader(fd)
-	var buf []byte
-	nr := 0
-	for {
-		var n uint32
-		if err := binary.Read(br, binary.BigEndian, &n); err != nil {
-			if errors.Is(err, io.EOF) {
-				break
+	for range trigger {
+		t0 := time.Now()
+		nowNanos := t0.UnixNano()
+		cutoff24h := t0.Add(-24 * time.Hour).UnixNano()
+		cutoff1w := t0.Add(-7 * 24 * time.Hour).UnixNano()
+		cutoff2Mon := t0.Add(-60 * 24 * time.Hour).UnixNano()
+		current, last24h, last1w, inactive, errors := 0, 0, 0, 0, 0
+
+		iter := s.db.NewIterator(&util.Range{}, nil)
+		for iter.Next() {
+			// Attempt to unmarshal the record and count the
+			// failure if there's something wrong with it.
+			var rec DatabaseRecord
+			if err := rec.Unmarshal(iter.Value()); err != nil {
+				errors++
+				continue
+			}
+
+			// If there are addresses that have not expired it's a current
+			// record, otherwise account it based on when it was last seen
+			// (last 24 hours or last week) or finally as inactice.
+			switch {
+			case len(expire(rec.Addresses, nowNanos)) > 0:
+				current++
+			case rec.Seen > cutoff24h:
+				last24h++
+			case rec.Seen > cutoff1w:
+				last1w++
+			case rec.Seen > cutoff2Mon:
+				inactive++
+			case rec.Missed < cutoff2Mon:
+				// It hasn't been seen lately and we haven't recorded
+				// someone asking for this device in a long time either;
+				// delete the record.
+				if err := s.db.Delete(iter.Key(), nil); err != nil {
+					databaseOperations.WithLabelValues(dbOpDelete, dbResError).Inc()
+				} else {
+					databaseOperations.WithLabelValues(dbOpDelete, dbResSuccess).Inc()
+				}
+			default:
+				inactive++
 			}
-			return nr, err
-		}
-		if int(n) > len(buf) {
-			buf = make([]byte, n)
-		}
-		if _, err := io.ReadFull(br, buf[:n]); err != nil {
-			return nr, err
-		}
-		rec := &discosrv.ReplicationRecord{}
-		if err := proto.Unmarshal(buf[:n], rec); err != nil {
-			return nr, err
-		}
-		key, err := protocol.DeviceIDFromBytes(rec.Key)
-		if err != nil {
-			key, err = protocol.DeviceIDFromString(string(rec.Key))
-		}
-		if err != nil {
-			log.Println("Bad device ID:", err)
-			continue
 		}
 
-		slices.SortFunc(rec.Addresses, Cmp)
-		rec.Addresses = slices.CompactFunc(rec.Addresses, Equal)
-		s.m.Store(key, &discosrv.DatabaseRecord{
-			Addresses: expire(rec.Addresses, s.clock.Now()),
-			Seen:      rec.Seen,
-		})
-		nr++
+		iter.Release()
+
+		databaseKeys.WithLabelValues("current").Set(float64(current))
+		databaseKeys.WithLabelValues("last24h").Set(float64(last24h))
+		databaseKeys.WithLabelValues("last1w").Set(float64(last1w))
+		databaseKeys.WithLabelValues("inactive").Set(float64(inactive))
+		databaseKeys.WithLabelValues("error").Set(float64(errors))
+		databaseStatisticsSeconds.Set(time.Since(t0).Seconds())
+
+		// Signal that we are done and can be scheduled again.
+		done <- struct{}{}
 	}
-	return nr, nil
 }
 
 // merge returns the merged result of the two database records a and b. The
 // result is the union of the two address sets, with the newer expiry time
-// chosen for any duplicates. The address list in a is overwritten and
-// reused for the result.
-func merge(a, b *discosrv.DatabaseRecord) *discosrv.DatabaseRecord {
+// chosen for any duplicates.
+func merge(a, b DatabaseRecord) DatabaseRecord {
 	// Both lists must be sorted for this to work.
+	if !sort.IsSorted(databaseAddressOrder(a.Addresses)) {
+		log.Println("Warning: bug: addresses not correctly sorted in merge")
+		a.Addresses = sortedAddressCopy(a.Addresses)
+	}
+	if !sort.IsSorted(databaseAddressOrder(b.Addresses)) {
+		// no warning because this is the side we read from disk and it may
+		// legitimately predate correct sorting.
+		b.Addresses = sortedAddressCopy(b.Addresses)
+	}
 
-	a.Seen = max(a.Seen, b.Seen)
+	res := DatabaseRecord{
+		Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
+		Seen:      a.Seen,
+	}
+	if b.Seen > a.Seen {
+		res.Seen = b.Seen
+	}
 
 	aIdx := 0
 	bIdx := 0
-	for aIdx < len(a.Addresses) && bIdx < len(b.Addresses) {
-		switch cmp.Compare(a.Addresses[aIdx].Address, b.Addresses[bIdx].Address) {
-		case 0:
-			// a == b, choose the newer expiry time
-			a.Addresses[aIdx].Expires = max(a.Addresses[aIdx].Expires, b.Addresses[bIdx].Expires)
+	aAddrs := a.Addresses
+	bAddrs := b.Addresses
+loop:
+	for {
+		switch {
+		case aIdx == len(aAddrs) && bIdx == len(bAddrs):
+			// both lists are exhausted, we are done
+			break loop
+
+		case aIdx == len(aAddrs):
+			// a is exhausted, pick from b and continue
+			res.Addresses = append(res.Addresses, bAddrs[bIdx])
+			bIdx++
+			continue
+
+		case bIdx == len(bAddrs):
+			// b is exhausted, pick from a and continue
+			res.Addresses = append(res.Addresses, aAddrs[aIdx])
+			aIdx++
+			continue
+		}
+
+		// We have values left on both sides.
+		aVal := aAddrs[aIdx]
+		bVal := bAddrs[bIdx]
+
+		switch {
+		case aVal.Address == bVal.Address:
+			// update for same address, pick newer
+			if aVal.Expires > bVal.Expires {
+				res.Addresses = append(res.Addresses, aVal)
+			} else {
+				res.Addresses = append(res.Addresses, bVal)
+			}
 			aIdx++
 			bIdx++
-		case -1:
-			// a < b, keep a and move on
+
+		case aVal.Address < bVal.Address:
+			// a is smallest, pick it and continue
+			res.Addresses = append(res.Addresses, aVal)
 			aIdx++
-		case 1:
-			// a > b, insert b before a
-			a.Addresses = append(a.Addresses[:aIdx], append([]*discosrv.DatabaseAddress{b.Addresses[bIdx]}, a.Addresses[aIdx:]...)...)
+
+		default:
+			// b is smallest, pick it and continue
+			res.Addresses = append(res.Addresses, bVal)
 			bIdx++
 		}
 	}
-	if bIdx < len(b.Addresses) {
-		a.Addresses = append(a.Addresses, b.Addresses[bIdx:]...)
-	}
-
-	return a
+	return res
 }
 
 // expire returns the list of addresses after removing expired entries.
 // Expiration happen in place, so the slice given as the parameter is
-// destroyed. Internal order is preserved.
-func expire(addrs []*discosrv.DatabaseAddress, now time.Time) []*discosrv.DatabaseAddress {
-	cutoff := now.UnixNano()
-	naddrs := addrs[:0]
-	for i := range addrs {
-		if i > 0 && addrs[i].Address == addrs[i-1].Address {
-			// Skip duplicates
+// destroyed. Internal order is not preserved.
+func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
+	i := 0
+	for i < len(addrs) {
+		if addrs[i].Expires < now {
+			// This item is expired. Replace it with the last in the list
+			// (noop if we are at the last item).
+			addrs[i] = addrs[len(addrs)-1]
+			// Wipe the last item of the list to release references to
+			// strings and stuff.
+			addrs[len(addrs)-1] = DatabaseAddress{}
+			// Shorten the slice.
+			addrs = addrs[:len(addrs)-1]
 			continue
 		}
-		if addrs[i].Expires >= cutoff {
-			naddrs = append(naddrs, addrs[i])
-		}
+		i++
 	}
-	if len(naddrs) == 0 {
-		return nil
-	}
-	return naddrs
+	return addrs
 }
 
-func Cmp(d, other *discosrv.DatabaseAddress) (n int) {
-	if c := cmp.Compare(d.Address, other.Address); c != 0 {
-		return c
-	}
-	return cmp.Compare(d.Expires, other.Expires)
+func sortedAddressCopy(addrs []DatabaseAddress) []DatabaseAddress {
+	sorted := make([]DatabaseAddress, len(addrs))
+	copy(sorted, addrs)
+	sort.Sort(databaseAddressOrder(sorted))
+	return sorted
 }
 
-func Equal(d, other *discosrv.DatabaseAddress) bool {
-	return d.Address == other.Address
+type databaseAddressOrder []DatabaseAddress
+
+func (s databaseAddressOrder) Less(a, b int) bool {
+	return s[a].Address < s[b].Address
+}
+
+func (s databaseAddressOrder) Swap(a, b int) {
+	s[a], s[b] = s[b], s[a]
+}
+
+func (s databaseAddressOrder) Len() int {
+	return len(s)
 }

cmd/stdiscosrv/database.pb.go

@@ -0,0 +1,847 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: database.proto
+
+package main
+
+import (
+	fmt "fmt"
+	_ "github.com/gogo/protobuf/gogoproto"
+	proto "github.com/gogo/protobuf/proto"
+	io "io"
+	math "math"
+	math_bits "math/bits"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+
+type DatabaseRecord struct {
+	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
+	Misses    int32             `protobuf:"varint,2,opt,name=misses,proto3" json:"misses,omitempty"`
+	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
+	Missed    int64             `protobuf:"varint,4,opt,name=missed,proto3" json:"missed,omitempty"`
+}
+
+func (m *DatabaseRecord) Reset()         { *m = DatabaseRecord{} }
+func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
+func (*DatabaseRecord) ProtoMessage()    {}
+func (*DatabaseRecord) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{0}
+}
+func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseRecord.Merge(m, src)
+}
+func (m *DatabaseRecord) XXX_Size() int {
+	return m.Size()
+}
+func (m *DatabaseRecord) XXX_DiscardUnknown() {
+	xxx_messageInfo_DatabaseRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DatabaseRecord proto.InternalMessageInfo
+
+type ReplicationRecord struct {
+	Key       string            `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+	Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses"`
+	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
+}
+
+func (m *ReplicationRecord) Reset()         { *m = ReplicationRecord{} }
+func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
+func (*ReplicationRecord) ProtoMessage()    {}
+func (*ReplicationRecord) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{1}
+}
+func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ReplicationRecord.Merge(m, src)
+}
+func (m *ReplicationRecord) XXX_Size() int {
+	return m.Size()
+}
+func (m *ReplicationRecord) XXX_DiscardUnknown() {
+	xxx_messageInfo_ReplicationRecord.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ReplicationRecord proto.InternalMessageInfo
+
+type DatabaseAddress struct {
+	Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
+	Expires int64  `protobuf:"varint,2,opt,name=expires,proto3" json:"expires,omitempty"`
+}
+
+func (m *DatabaseAddress) Reset()         { *m = DatabaseAddress{} }
+func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
+func (*DatabaseAddress) ProtoMessage()    {}
+func (*DatabaseAddress) Descriptor() ([]byte, []int) {
+	return fileDescriptor_b90fe3356ea5df07, []int{2}
+}
+func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseAddress.Merge(m, src)
+}
+func (m *DatabaseAddress) XXX_Size() int {
+	return m.Size()
+}
+func (m *DatabaseAddress) XXX_DiscardUnknown() {
+	xxx_messageInfo_DatabaseAddress.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DatabaseAddress proto.InternalMessageInfo
+
+func init() {
+	proto.RegisterType((*DatabaseRecord)(nil), "main.DatabaseRecord")
+	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
+	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
+}
+
+func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
+
+var fileDescriptor_b90fe3356ea5df07 = []byte{
+	// 270 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
+	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
+	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
+	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
+	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
+	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
+	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
+	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
+	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
+	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
+	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
+	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
+	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
+	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
+	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
+	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
+	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
+}
+
+func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Missed != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+		i--
+		dAtA[i] = 0x20
+	}
+	if m.Seen != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
+	}
+	if m.Misses != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+		i--
+		dAtA[i] = 0x10
+	}
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Seen != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
+	}
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x12
+		}
+	}
+	if len(m.Key) > 0 {
+		i -= len(m.Key)
+		copy(dAtA[i:], m.Key)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Expires != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
+		i--
+		dAtA[i] = 0x10
+	}
+	if len(m.Address) > 0 {
+		i -= len(m.Address)
+		copy(dAtA[i:], m.Address)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
+	offset -= sovDatabase(v)
+	base := offset
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return base
+}
+func (m *DatabaseRecord) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Addresses) > 0 {
+		for _, e := range m.Addresses {
+			l = e.Size()
+			n += 1 + l + sovDatabase(uint64(l))
+		}
+	}
+	if m.Misses != 0 {
+		n += 1 + sovDatabase(uint64(m.Misses))
+	}
+	if m.Seen != 0 {
+		n += 1 + sovDatabase(uint64(m.Seen))
+	}
+	if m.Missed != 0 {
+		n += 1 + sovDatabase(uint64(m.Missed))
+	}
+	return n
+}
+
+func (m *ReplicationRecord) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Key)
+	if l > 0 {
+		n += 1 + l + sovDatabase(uint64(l))
+	}
+	if len(m.Addresses) > 0 {
+		for _, e := range m.Addresses {
+			l = e.Size()
+			n += 1 + l + sovDatabase(uint64(l))
+		}
+	}
+	if m.Seen != 0 {
+		n += 1 + sovDatabase(uint64(m.Seen))
+	}
+	return n
+}
+
+func (m *DatabaseAddress) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Address)
+	if l > 0 {
+		n += 1 + l + sovDatabase(uint64(l))
+	}
+	if m.Expires != 0 {
+		n += 1 + sovDatabase(uint64(m.Expires))
+	}
+	return n
+}
+
+func sovDatabase(x uint64) (n int) {
+	return (math_bits.Len64(x|1) + 6) / 7
+}
+func sozDatabase(x uint64) (n int) {
+	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: DatabaseRecord: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: DatabaseRecord: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Addresses = append(m.Addresses, DatabaseAddress{})
+			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Misses", wireType)
+			}
+			m.Misses = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Misses |= int32(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
+			}
+			m.Seen = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Seen |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Missed", wireType)
+			}
+			m.Missed = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Missed |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ReplicationRecord: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ReplicationRecord: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Key = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Addresses = append(m.Addresses, DatabaseAddress{})
+			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
+			}
+			m.Seen = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Seen |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: DatabaseAddress: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: DatabaseAddress: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Address", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Address = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Expires", wireType)
+			}
+			m.Expires = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Expires |= int64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipDatabase(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipDatabase(dAtA []byte) (n int, err error) {
+	l := len(dAtA)
+	iNdEx := 0
+	depth := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowDatabase
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if dAtA[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+		case 1:
+			iNdEx += 8
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowDatabase
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if length < 0 {
+				return 0, ErrInvalidLengthDatabase
+			}
+			iNdEx += length
+		case 3:
+			depth++
+		case 4:
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupDatabase
+			}
+			depth--
+		case 5:
+			iNdEx += 4
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthDatabase
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
+	}
+	return 0, io.ErrUnexpectedEOF
+}
+
+var (
+	ErrInvalidLengthDatabase        = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowDatabase          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupDatabase = fmt.Errorf("proto: unexpected end of group")
+)

cmd/stdiscosrv/database.proto

@@ -0,0 +1,36 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+syntax = "proto3";
+
+package main;
+
+import "repos/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.goproto_unkeyed_all) = false;
+option (gogoproto.goproto_unrecognized_all) = false;
+option (gogoproto.goproto_sizecache_all) = false;
+
+message DatabaseRecord {
+    repeated DatabaseAddress addresses = 1 [(gogoproto.nullable) = false];
+    int32                    misses    = 2; // Number of lookups* without hits
+    int64                    seen      = 3; // Unix nanos, last device announce
+    int64                    missed    = 4; // Unix nanos, last* failed lookup
+}
+
+// *) Not every lookup results in a write, so may not be completely accurate
+
+message ReplicationRecord {
+    string                   key       = 1;
+    repeated DatabaseAddress addresses = 2 [(gogoproto.nullable) = false];
+    int64                    seen      = 3; // Unix nanos, last device announce
+}
+
+message DatabaseAddress {
+    string address = 1;
+    int64  expires = 2; // Unix nanos
+}

cmd/stdiscosrv/database_test.go

@@ -9,28 +9,34 @@ package main
 import (
 	"context"
 	"fmt"
+	"os"
 	"testing"
 	"time"
-
-	"github.com/syncthing/syncthing/internal/gen/discosrv"
-	"github.com/syncthing/syncthing/lib/protocol"
 )
 
 func TestDatabaseGetSet(t *testing.T) {
-	db := newInMemoryStore(t.TempDir(), 0, nil)
+	os.RemoveAll("_database")
+	defer os.RemoveAll("_database")
+	db, err := newLevelDBStore("_database")
+	if err != nil {
+		t.Fatal(err)
+	}
 	ctx, cancel := context.WithCancel(context.Background())
 	go db.Serve(ctx)
 	defer cancel()
 
 	// Check missing record
 
-	rec, err := db.get(&protocol.EmptyDeviceID)
+	rec, err := db.get("abcd")
 	if err != nil {
 		t.Error("not found should not be an error")
 	}
 	if len(rec.Addresses) != 0 {
 		t.Error("addresses should be empty")
 	}
+	if rec.Misses != 0 {
+		t.Error("missing should be zero")
+	}
 
 	// Set up a clock
 
@@ -40,16 +46,16 @@ func TestDatabaseGetSet(t *testing.T) {
 
 	// Put a record
 
-	rec.Addresses = []*discosrv.DatabaseAddress{
+	rec.Addresses = []DatabaseAddress{
 		{Address: "tcp://1.2.3.4:5", Expires: tc.Now().Add(time.Minute).UnixNano()},
 	}
-	if err := db.put(&protocol.EmptyDeviceID, rec); err != nil {
+	if err := db.put("abcd", rec); err != nil {
 		t.Fatal(err)
 	}
 
 	// Verify it
 
-	rec, err = db.get(&protocol.EmptyDeviceID)
+	rec, err = db.get("abcd")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -66,16 +72,16 @@ func TestDatabaseGetSet(t *testing.T) {
 
 	tc.wind(30 * time.Second)
 
-	addrs := []*discosrv.DatabaseAddress{
+	addrs := []DatabaseAddress{
 		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
 	}
-	if err := db.merge(&protocol.EmptyDeviceID, addrs, tc.Now().UnixNano()); err != nil {
+	if err := db.merge("abcd", addrs, tc.Now().UnixNano()); err != nil {
 		t.Fatal(err)
 	}
 
 	// Verify it
 
-	rec, err = db.get(&protocol.EmptyDeviceID)
+	rec, err = db.get("abcd")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -98,7 +104,7 @@ func TestDatabaseGetSet(t *testing.T) {
 
 	// Verify it
 
-	rec, err = db.get(&protocol.EmptyDeviceID)
+	rec, err = db.get("abcd")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -111,18 +117,40 @@ func TestDatabaseGetSet(t *testing.T) {
 		t.Error("incorrect address")
 	}
 
-	// Set an address
+	// Put a record with misses
 
-	addrs = []*discosrv.DatabaseAddress{
-		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
-	}
-	if err := db.merge(&protocol.GlobalDeviceID, addrs, tc.Now().UnixNano()); err != nil {
+	rec = DatabaseRecord{Misses: 42}
+	if err := db.put("efgh", rec); err != nil {
 		t.Fatal(err)
 	}
 
 	// Verify it
 
-	rec, err = db.get(&protocol.GlobalDeviceID)
+	rec, err = db.get("efgh")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rec.Addresses) != 0 {
+		t.Log(rec.Addresses)
+		t.Fatal("should have no addresses")
+	}
+	if rec.Misses != 42 {
+		t.Log(rec.Misses)
+		t.Error("incorrect misses")
+	}
+
+	// Set an address
+
+	addrs = []DatabaseAddress{
+		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
+	}
+	if err := db.merge("efgh", addrs, tc.Now().UnixNano()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify it
+
+	rec, err = db.get("efgh")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -130,126 +158,48 @@ func TestDatabaseGetSet(t *testing.T) {
 		t.Log(rec.Addresses)
 		t.Fatal("should have one address")
 	}
+	if rec.Misses != 0 {
+		t.Log(rec.Misses)
+		t.Error("should have no misses")
+	}
 }
 
 func TestFilter(t *testing.T) {
 	// all cases are expired with t=10
 	cases := []struct {
-		a []*discosrv.DatabaseAddress
-		b []*discosrv.DatabaseAddress
+		a []DatabaseAddress
+		b []DatabaseAddress
 	}{
 		{
 			a: nil,
 			b: nil,
 		},
 		{
-			a: []*discosrv.DatabaseAddress{{Address: "a", Expires: 9}, {Address: "b", Expires: 9}, {Address: "c", Expires: 9}},
-			b: []*discosrv.DatabaseAddress{},
+			a: []DatabaseAddress{{Address: "a", Expires: 9}, {Address: "b", Expires: 9}, {Address: "c", Expires: 9}},
+			b: []DatabaseAddress{},
 		},
 		{
-			a: []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
-			b: []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
+			a: []DatabaseAddress{{Address: "a", Expires: 10}},
+			b: []DatabaseAddress{{Address: "a", Expires: 10}},
 		},
 		{
-			a: []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
-			b: []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
+			a: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
+			b: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
 		},
 		{
-			a: []*discosrv.DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
-			b: []*discosrv.DatabaseAddress{{Address: "b", Expires: 15}, {Address: "d", Expires: 15}},
+			a: []DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
+			b: []DatabaseAddress{{Address: "d", Expires: 15}, {Address: "b", Expires: 15}}, // gets reordered
 		},
 	}
 
 	for _, tc := range cases {
-		res := expire(tc.a, time.Unix(0, 10))
+		res := expire(tc.a, 10)
 		if fmt.Sprint(res) != fmt.Sprint(tc.b) {
 			t.Errorf("Incorrect result %v, expected %v", res, tc.b)
 		}
 	}
 }
 
-func TestMerge(t *testing.T) {
-	cases := []struct {
-		a, b, res []*discosrv.DatabaseAddress
-	}{
-		{nil, nil, nil},
-		{
-			nil,
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
-		},
-		{
-			nil,
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
-		},
-		{
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 15}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 15}},
-		},
-		{
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}},
-			[]*discosrv.DatabaseAddress{{Address: "b", Expires: 15}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
-		},
-		{
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 15}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 15}},
-		},
-		{
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
-			[]*discosrv.DatabaseAddress{{Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
-		},
-		{
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}},
-			[]*discosrv.DatabaseAddress{{Address: "b", Expires: 5}, {Address: "c", Expires: 20}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}},
-		},
-		{
-			[]*discosrv.DatabaseAddress{{Address: "y", Expires: 10}, {Address: "z", Expires: 10}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "y", Expires: 10}, {Address: "z", Expires: 10}},
-		},
-		{
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "d", Expires: 10}},
-			[]*discosrv.DatabaseAddress{{Address: "b", Expires: 5}, {Address: "c", Expires: 20}},
-			[]*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}, {Address: "c", Expires: 20}, {Address: "d", Expires: 10}},
-		},
-	}
-
-	for _, tc := range cases {
-		rec := merge(&discosrv.DatabaseRecord{Addresses: tc.a}, &discosrv.DatabaseRecord{Addresses: tc.b})
-		if fmt.Sprint(rec.Addresses) != fmt.Sprint(tc.res) {
-			t.Errorf("Incorrect result %v, expected %v", rec.Addresses, tc.res)
-		}
-		rec = merge(&discosrv.DatabaseRecord{Addresses: tc.b}, &discosrv.DatabaseRecord{Addresses: tc.a})
-		if fmt.Sprint(rec.Addresses) != fmt.Sprint(tc.res) {
-			t.Errorf("Incorrect result %v, expected %v", rec.Addresses, tc.res)
-		}
-	}
-}
-
-func BenchmarkMergeEqual(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		ar := []*discosrv.DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 15}}
-		br := []*discosrv.DatabaseAddress{{Address: "a", Expires: 15}, {Address: "b", Expires: 10}}
-		res := merge(&discosrv.DatabaseRecord{Addresses: ar}, &discosrv.DatabaseRecord{Addresses: br})
-		if len(res.Addresses) != 2 {
-			b.Fatal("wrong length")
-		}
-		if res.Addresses[0].Address != "a" || res.Addresses[1].Address != "b" {
-			b.Fatal("wrong address")
-		}
-		if res.Addresses[0].Expires != 15 || res.Addresses[1].Expires != 15 {
-			b.Fatal("wrong expiry")
-		}
-	}
-	b.ReportAllocs() // should be zero per operation
-}
-
 type testClock struct {
 	now time.Time
 }
@@ -259,6 +209,5 @@ func (t *testClock) wind(d time.Duration) {
 }
 
 func (t *testClock) Now() time.Time {
-	t.now = t.now.Add(time.Nanosecond)
 	return t.now
 }

cmd/stdiscosrv/main.go

@@ -9,24 +9,20 @@ package main
 import (
 	"context"
 	"crypto/tls"
+	"flag"
 	"log"
+	"net"
 	"net/http"
-	_ "net/http/pprof"
 	"os"
-	"os/signal"
-	"runtime"
+	"strings"
 	"time"
 
-	"github.com/alecthomas/kong"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/thejerf/suture/v4"
-
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/rand"
-	"github.com/syncthing/syncthing/lib/s3"
 	"github.com/syncthing/syncthing/lib/tlsutil"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/thejerf/suture/v4"
 )
 
 const (
@@ -40,12 +36,17 @@ const (
 	errorRetryAfterSeconds = 1500
 	errorRetryFuzzSeconds  = 300
 
-	// Retry for not found is notFoundRetrySeenSeconds for records we have
-	// seen an announcement for (but it's not active right now) and
-	// notFoundRetryUnknownSeconds for records we have never seen (or not
-	// seen within the last week).
-	notFoundRetryUnknownMinSeconds = 60
-	notFoundRetryUnknownMaxSeconds = 3600
+	// Retry for not found is minSeconds + failures * incSeconds +
+	// random(fuzz), where failures is the number of consecutive lookups
+	// with no answer, up to maxSeconds. The fuzz is applied after capping
+	// to maxSeconds.
+	notFoundRetryMinSeconds  = 60
+	notFoundRetryMaxSeconds  = 3540
+	notFoundRetryIncSeconds  = 10
+	notFoundRetryFuzzSeconds = 60
+
+	// How often (in requests) we serialize the missed counter to database.
+	notFoundMissesWriteInterval = 10
 
 	httpReadTimeout    = 5 * time.Second
 	httpWriteTimeout   = 5 * time.Second
@@ -55,116 +56,133 @@ const (
 	replicationOutboxSize = 10000
 )
 
-var debug = false
-
-type CLI struct {
-	Cert          string `group:"Listen" help:"Certificate file" default:"./cert.pem" env:"DISCOVERY_CERT_FILE"`
-	Key           string `group:"Listen" help:"Key file" default:"./key.pem" env:"DISCOVERY_KEY_FILE"`
-	HTTP          bool   `group:"Listen" help:"Listen on HTTP (behind an HTTPS proxy)" env:"DISCOVERY_HTTP"`
-	Compression   bool   `group:"Listen" help:"Enable GZIP compression of responses" env:"DISCOVERY_COMPRESSION"`
-	Listen        string `group:"Listen" help:"Listen address" default:":8443" env:"DISCOVERY_LISTEN"`
-	MetricsListen string `group:"Listen" help:"Metrics listen address" env:"DISCOVERY_METRICS_LISTEN"`
-
-	DBDir           string        `group:"Database" help:"Database directory" default:"." env:"DISCOVERY_DB_DIR"`
-	DBFlushInterval time.Duration `group:"Database" help:"Interval between database flushes" default:"5m" env:"DISCOVERY_DB_FLUSH_INTERVAL"`
-
-	DBS3Endpoint    string `name:"db-s3-endpoint" group:"Database (S3 backup)" hidden:"true" help:"S3 endpoint for database" env:"DISCOVERY_DB_S3_ENDPOINT"`
-	DBS3Region      string `name:"db-s3-region" group:"Database (S3 backup)" hidden:"true" help:"S3 region for database" env:"DISCOVERY_DB_S3_REGION"`
-	DBS3Bucket      string `name:"db-s3-bucket" group:"Database (S3 backup)" hidden:"true" help:"S3 bucket for database" env:"DISCOVERY_DB_S3_BUCKET"`
-	DBS3AccessKeyID string `name:"db-s3-access-key-id" group:"Database (S3 backup)" hidden:"true" help:"S3 access key ID for database" env:"DISCOVERY_DB_S3_ACCESS_KEY_ID"`
-	DBS3SecretKey   string `name:"db-s3-secret-key" group:"Database (S3 backup)" hidden:"true" help:"S3 secret key for database" env:"DISCOVERY_DB_S3_SECRET_KEY"`
-
-	AMQPAddress string `group:"AMQP replication" hidden:"true" help:"Address to AMQP broker" env:"DISCOVERY_AMQP_ADDRESS"`
-
-	Debug   bool `short:"d" help:"Print debug output" env:"DISCOVERY_DEBUG"`
-	Version bool `short:"v" help:"Print version and exit"`
+// These options make the database a little more optimized for writes, at
+// the expense of some memory usage and risk of losing writes in a (system)
+// crash.
+var levelDBOptions = &opt.Options{
+	NoSync:      true,
+	WriteBuffer: 32 << 20, // default 4<<20
 }
 
-func main() {
-	log.SetOutput(os.Stdout)
+var (
+	debug = false
+)
 
-	var cli CLI
-	kong.Parse(&cli)
-	debug = cli.Debug
+func main() {
+	var listen string
+	var dir string
+	var metricsListen string
+	var replicationListen string
+	var replicationPeers string
+	var certFile string
+	var keyFile string
+	var useHTTP bool
+
+	log.SetOutput(os.Stdout)
+	log.SetFlags(0)
+
+	flag.StringVar(&certFile, "cert", "./cert.pem", "Certificate file")
+	flag.StringVar(&dir, "db-dir", "./discovery.db", "Database directory")
+	flag.BoolVar(&debug, "debug", false, "Print debug output")
+	flag.BoolVar(&useHTTP, "http", false, "Listen on HTTP (behind an HTTPS proxy)")
+	flag.StringVar(&listen, "listen", ":8443", "Listen address")
+	flag.StringVar(&keyFile, "key", "./key.pem", "Key file")
+	flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
+	flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
+	flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
+	showVersion := flag.Bool("version", false, "Show version")
+	flag.Parse()
 
 	log.Println(build.LongVersionFor("stdiscosrv"))
-	if cli.Version {
+	if *showVersion {
 		return
 	}
 
-	buildInfo.WithLabelValues(build.Version, runtime.Version(), build.User, build.Date.UTC().Format("2006-01-02T15:04:05Z")).Set(1)
-
-	var cert tls.Certificate
-	if !cli.HTTP {
-		var err error
-		cert, err = tls.LoadX509KeyPair(cli.Cert, cli.Key)
-		if os.IsNotExist(err) {
-			log.Println("Failed to load keypair. Generating one, this might take a while...")
-			cert, err = tlsutil.NewCertificate(cli.Cert, cli.Key, "stdiscosrv", 20*365)
-			if err != nil {
-				log.Fatalln("Failed to generate X509 key pair:", err)
-			}
-		} else if err != nil {
-			log.Fatalln("Failed to load keypair:", err)
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if os.IsNotExist(err) {
+		log.Println("Failed to load keypair. Generating one, this might take a while...")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 20*365)
+		if err != nil {
+			log.Fatalln("Failed to generate X509 key pair:", err)
+		}
+	} else if err != nil {
+		log.Fatalln("Failed to load keypair:", err)
+	}
+	devID := protocol.NewDeviceID(cert.Certificate[0])
+	log.Println("Server device ID is", devID)
+
+	// Parse the replication specs, if any.
+	var allowedReplicationPeers []protocol.DeviceID
+	var replicationDestinations []string
+	parts := strings.Split(replicationPeers, ",")
+	for _, part := range parts {
+		fields := strings.Split(part, "@")
+
+		switch len(fields) {
+		case 2:
+			// This is an id@address specification. Grab the address for the
+			// destination list. Try to resolve it once to catch obvious
+			// syntax errors here rather than having the sender service fail
+			// repeatedly later.
+			_, err := net.ResolveTCPAddr("tcp", fields[1])
+			if err != nil {
+				log.Fatalln("Resolving address:", err)
+			}
+			replicationDestinations = append(replicationDestinations, fields[1])
+			fallthrough // N.B.
+
+		case 1:
+			// The first part is always a device ID.
+			id, err := protocol.DeviceIDFromString(fields[0])
+			if err != nil {
+				log.Fatalln("Parsing device ID:", err)
+			}
+			allowedReplicationPeers = append(allowedReplicationPeers, id)
+
+		default:
+			log.Fatalln("Unrecognized replication spec:", part)
 		}
-		devID := protocol.NewDeviceID(cert.Certificate[0])
-		log.Println("Server device ID is", devID)
 	}
 
 	// Root of the service tree.
 	main := suture.New("main", suture.Spec{
 		PassThroughPanics: true,
-		Timeout:           2 * time.Minute,
 	})
 
-	// If configured, use S3 for database backups.
-	var s3c *s3.Session
-	if cli.DBS3Endpoint != "" {
-		var err error
-		s3c, err = s3.NewSession(cli.DBS3Endpoint, cli.DBS3Region, cli.DBS3Bucket, cli.DBS3AccessKeyID, cli.DBS3SecretKey)
-		if err != nil {
-			log.Fatalf("Failed to create S3 session: %v", err)
-		}
-	}
-
 	// Start the database.
-	db := newInMemoryStore(cli.DBDir, cli.DBFlushInterval, s3c)
+	db, err := newLevelDBStore(dir)
+	if err != nil {
+		log.Fatalln("Open database:", err)
+	}
 	main.Add(db)
 
-	// If we have an AMQP broker for replication, start that
-	var repl replicator
-	if cli.AMQPAddress != "" {
-		clientID := rand.String(10)
-		kr := newAMQPReplicator(cli.AMQPAddress, clientID, db)
-		main.Add(kr)
-		repl = kr
+	// Start any replication senders.
+	var repl replicationMultiplexer
+	for _, dst := range replicationDestinations {
+		rs := newReplicationSender(dst, cert, allowedReplicationPeers)
+		main.Add(rs)
+		repl = append(repl, rs)
+	}
+
+	// If we have replication configured, start the replication listener.
+	if len(allowedReplicationPeers) > 0 {
+		rl := newReplicationListener(replicationListen, cert, allowedReplicationPeers, db)
+		main.Add(rl)
 	}
 
 	// Start the main API server.
-	qs := newAPISrv(cli.Listen, cert, db, repl, cli.HTTP, cli.Compression)
+	qs := newAPISrv(listen, cert, db, repl, useHTTP)
 	main.Add(qs)
 
 	// If we have a metrics port configured, start a metrics handler.
-	if cli.MetricsListen != "" {
+	if metricsListen != "" {
 		go func() {
 			mux := http.NewServeMux()
 			mux.Handle("/metrics", promhttp.Handler())
-			log.Fatal(http.ListenAndServe(cli.MetricsListen, mux))
+			log.Fatal(http.ListenAndServe(metricsListen, mux))
 		}()
 	}
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	// Cancel on signal
-	signalChan := make(chan os.Signal, 1)
-	signal.Notify(signalChan, os.Interrupt)
-	go func() {
-		sig := <-signalChan
-		log.Printf("Received signal %s; shutting down", sig)
-		cancel()
-	}()
-
 	// Engage!
-	main.Serve(ctx)
+	main.Serve(context.Background())
 }

cmd/stdiscosrv/replication.go

@@ -0,0 +1,315 @@
+// Copyright (C) 2018 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/binary"
+	"fmt"
+	io "io"
+	"log"
+	"net"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+const replicationReadTimeout = time.Minute
+const replicationHeartbeatInterval = time.Second * 30
+
+type replicator interface {
+	send(key string, addrs []DatabaseAddress, seen int64)
+}
+
+// a replicationSender tries to connect to the remote address and provide
+// them with a feed of replication updates.
+type replicationSender struct {
+	dst        string
+	cert       tls.Certificate // our certificate
+	allowedIDs []protocol.DeviceID
+	outbox     chan ReplicationRecord
+}
+
+func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
+	return &replicationSender{
+		dst:        dst,
+		cert:       cert,
+		allowedIDs: allowedIDs,
+		outbox:     make(chan ReplicationRecord, replicationOutboxSize),
+	}
+}
+
+func (s *replicationSender) Serve(ctx context.Context) error {
+	// Sleep a little at startup. Peers often restart at the same time, and
+	// this avoid the service failing and entering backoff state
+	// unnecessarily, while also reducing the reconnect rate to something
+	// reasonable by default.
+	time.Sleep(2 * time.Second)
+
+	tlsCfg := &tls.Config{
+		Certificates:       []tls.Certificate{s.cert},
+		MinVersion:         tls.VersionTLS12,
+		InsecureSkipVerify: true,
+	}
+
+	// Dial the TLS connection.
+	conn, err := tls.Dial("tcp", s.dst, tlsCfg)
+	if err != nil {
+		log.Println("Replication connect:", err)
+		return err
+	}
+	defer func() {
+		conn.SetWriteDeadline(time.Now().Add(time.Second))
+		conn.Close()
+	}()
+
+	// Get the other side device ID.
+	remoteID, err := deviceID(conn)
+	if err != nil {
+		log.Println("Replication connect:", err)
+		return err
+	}
+
+	// Verify it's in the set of allowed device IDs.
+	if !deviceIDIn(remoteID, s.allowedIDs) {
+		log.Println("Replication connect: unexpected device ID:", remoteID)
+		return err
+	}
+
+	heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
+	defer heartBeatTicker.Stop()
+
+	// Send records.
+	buf := make([]byte, 1024)
+	for {
+		select {
+		case <-heartBeatTicker.C:
+			if len(s.outbox) > 0 {
+				// No need to send heartbeats if there are events/prevrious
+				// heartbeats to send, they will keep the connection alive.
+				continue
+			}
+			// Empty replication message is the heartbeat:
+			s.outbox <- ReplicationRecord{}
+
+		case rec := <-s.outbox:
+			// Buffer must hold record plus four bytes for size
+			size := rec.Size()
+			if len(buf) < size+4 {
+				buf = make([]byte, size+4)
+			}
+
+			// Record comes after the four bytes size
+			n, err := rec.MarshalTo(buf[4:])
+			if err != nil {
+				// odd to get an error here, but we haven't sent anything
+				// yet so it's not fatal
+				replicationSendsTotal.WithLabelValues("error").Inc()
+				log.Println("Replication marshal:", err)
+				continue
+			}
+			binary.BigEndian.PutUint32(buf, uint32(n))
+
+			// Send
+			conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
+			if _, err := conn.Write(buf[:4+n]); err != nil {
+				replicationSendsTotal.WithLabelValues("error").Inc()
+				log.Println("Replication write:", err)
+				// Yes, we are loosing the replication event here.
+				return err
+			}
+			replicationSendsTotal.WithLabelValues("success").Inc()
+
+		case <-ctx.Done():
+			return nil
+		}
+	}
+}
+
+func (s *replicationSender) String() string {
+	return fmt.Sprintf("replicationSender(%q)", s.dst)
+}
+
+func (s *replicationSender) send(key string, ps []DatabaseAddress, seen int64) {
+	item := ReplicationRecord{
+		Key:       key,
+		Addresses: ps,
+	}
+
+	// The send should never block. The inbox is suitably buffered for at
+	// least a few seconds of stalls, which shouldn't happen in practice.
+	select {
+	case s.outbox <- item:
+	default:
+		replicationSendsTotal.WithLabelValues("drop").Inc()
+	}
+}
+
+// a replicationMultiplexer sends to multiple replicators
+type replicationMultiplexer []replicator
+
+func (m replicationMultiplexer) send(key string, ps []DatabaseAddress, seen int64) {
+	for _, s := range m {
+		// each send is nonblocking
+		s.send(key, ps, seen)
+	}
+}
+
+// replicationListener accepts incoming connections and reads replication
+// items from them. Incoming items are applied to the KV store.
+type replicationListener struct {
+	addr       string
+	cert       tls.Certificate
+	allowedIDs []protocol.DeviceID
+	db         database
+}
+
+func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
+	return &replicationListener{
+		addr:       addr,
+		cert:       cert,
+		allowedIDs: allowedIDs,
+		db:         db,
+	}
+}
+
+func (l *replicationListener) Serve(ctx context.Context) error {
+	tlsCfg := &tls.Config{
+		Certificates:       []tls.Certificate{l.cert},
+		ClientAuth:         tls.RequestClientCert,
+		MinVersion:         tls.VersionTLS12,
+		InsecureSkipVerify: true,
+	}
+
+	lst, err := tls.Listen("tcp", l.addr, tlsCfg)
+	if err != nil {
+		log.Println("Replication listen:", err)
+		return err
+	}
+	defer lst.Close()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+		}
+
+		// Accept a connection
+		conn, err := lst.Accept()
+		if err != nil {
+			log.Println("Replication accept:", err)
+			return err
+		}
+
+		// Figure out the other side device ID
+		remoteID, err := deviceID(conn.(*tls.Conn))
+		if err != nil {
+			log.Println("Replication accept:", err)
+			conn.SetWriteDeadline(time.Now().Add(time.Second))
+			conn.Close()
+			continue
+		}
+
+		// Verify it is in the set of allowed device IDs
+		if !deviceIDIn(remoteID, l.allowedIDs) {
+			log.Println("Replication accept: unexpected device ID:", remoteID)
+			conn.SetWriteDeadline(time.Now().Add(time.Second))
+			conn.Close()
+			continue
+		}
+
+		go l.handle(ctx, conn)
+	}
+}
+
+func (l *replicationListener) String() string {
+	return fmt.Sprintf("replicationListener(%q)", l.addr)
+}
+
+func (l *replicationListener) handle(ctx context.Context, conn net.Conn) {
+	defer func() {
+		conn.SetWriteDeadline(time.Now().Add(time.Second))
+		conn.Close()
+	}()
+
+	buf := make([]byte, 1024)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		conn.SetReadDeadline(time.Now().Add(replicationReadTimeout))
+
+		// First four bytes are the size
+		if _, err := io.ReadFull(conn, buf[:4]); err != nil {
+			log.Println("Replication read size:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			return
+		}
+
+		// Read the rest of the record
+		size := int(binary.BigEndian.Uint32(buf[:4]))
+		if len(buf) < size {
+			buf = make([]byte, size)
+		}
+
+		if size == 0 {
+			// Heartbeat, ignore
+			continue
+		}
+
+		if _, err := io.ReadFull(conn, buf[:size]); err != nil {
+			log.Println("Replication read record:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			return
+		}
+
+		// Unmarshal
+		var rec ReplicationRecord
+		if err := rec.Unmarshal(buf[:size]); err != nil {
+			log.Println("Replication unmarshal:", err)
+			replicationRecvsTotal.WithLabelValues("error").Inc()
+			continue
+		}
+
+		// Store
+		l.db.merge(rec.Key, rec.Addresses, rec.Seen)
+		replicationRecvsTotal.WithLabelValues("success").Inc()
+	}
+}
+
+func deviceID(conn *tls.Conn) (protocol.DeviceID, error) {
+	// Handshake may not be complete on the server side yet, which we need
+	// to get the client certificate.
+	if !conn.ConnectionState().HandshakeComplete {
+		if err := conn.Handshake(); err != nil {
+			return protocol.DeviceID{}, err
+		}
+	}
+
+	// We expect exactly one certificate.
+	certs := conn.ConnectionState().PeerCertificates
+	if len(certs) != 1 {
+		return protocol.DeviceID{}, fmt.Errorf("unexpected number of certificates (%d != 1)", len(certs))
+	}
+
+	return protocol.NewDeviceID(certs[0].Raw), nil
+}
+
+func deviceIDIn(id protocol.DeviceID, ids []protocol.DeviceID) bool {
+	for _, candidate := range ids {
+		if id == candidate {
+			return true
+		}
+	}
+	return false
+}

cmd/stdiscosrv/stats.go

@@ -7,18 +7,12 @@
 package main
 
 import (
+	"os"
+
 	"github.com/prometheus/client_golang/prometheus"
 )
 
 var (
-	buildInfo = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "build_info",
-			Help:      "A metric with a constant '1' value labeled by version, goversion, builduser and builddate from which stdiscosrv was built.",
-		}, []string{"version", "goversion", "builduser", "builddate"})
-
 	apiRequestsTotal = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: "syncthing",
@@ -95,29 +89,6 @@ var (
 			Help:       "Latency of database operations.",
 			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
 		}, []string{"operation"})
-
-	databaseWriteSeconds = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "database_write_seconds",
-			Help:      "Time spent writing the database.",
-		})
-	databaseLastWritten = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "database_last_written",
-			Help:      "Timestamp of the last successful database write.",
-		})
-
-	retryAfterLevel = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "retry_after_seconds",
-			Help:      "Retry-After header value in seconds.",
-		}, []string{"name"})
 )
 
 const (
@@ -132,12 +103,21 @@ const (
 )
 
 func init() {
-	prometheus.MustRegister(buildInfo,
-		apiRequestsTotal, apiRequestsSeconds,
+	prometheus.MustRegister(apiRequestsTotal, apiRequestsSeconds,
 		lookupRequestsTotal, announceRequestsTotal,
 		replicationSendsTotal, replicationRecvsTotal,
 		databaseKeys, databaseStatisticsSeconds,
-		databaseOperations, databaseOperationSeconds,
-		databaseWriteSeconds, databaseLastWritten,
-		retryAfterLevel)
+		databaseOperations, databaseOperationSeconds)
+
+	processCollectorOpts := prometheus.ProcessCollectorOpts{
+		Namespace: "syncthing_discovery",
+		PidFn: func() (int, error) {
+			return os.Getpid(), nil
+		},
+	}
+
+	prometheus.MustRegister(
+		prometheus.NewProcessCollector(processCollectorOpts),
+	)
+
 }

cmd/stevents/main.go

@@ -14,8 +14,6 @@ import (
 	"net/http"
 	"os"
 	"time"
-
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 )
 
 type event struct {

cmd/stfileinfo/main.go

@@ -13,7 +13,6 @@ import (
 	"os"
 	"path/filepath"
 
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/scanner"
 )

cmd/stfinddevice/main.go

@@ -16,7 +16,6 @@ import (
 	"os"
 	"time"
 
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
 	"github.com/syncthing/syncthing/lib/events"
@@ -85,7 +84,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 }
 
 func checkServer(deviceID protocol.DeviceID, server string) checkResult {
-	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger, nil)
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger)
 	if err != nil {
 		return checkResult{error: err}
 	}

cmd/stfindignored/main.go

@@ -4,7 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// Command stfindignored lists ignored files under a given folder root.
+// Commmand stfindignored lists ignored files under a given folder root.
 package main
 
 import (
@@ -12,7 +12,6 @@ import (
 	"fmt"
 	"os"
 
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/ignore"
 )

cmd/stgenfiles/main.go

@@ -15,8 +15,6 @@ import (
 	"os"
 	"path/filepath"
 	"time"
-
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 )
 
 func main() {
@@ -45,7 +43,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
 		}
 
 		p0 := filepath.Join(dir, string(n[0]), n[0:2])
-		err = os.MkdirAll(p0, 0o755)
+		err = os.MkdirAll(p0, 0755)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -68,7 +66,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
 }
 
 func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
-	src := io.LimitReader(&infiniteReader{fd}, s)
+	src := io.LimitReader(&inifiteReader{fd}, s)
 	dst, err := os.Create(p1)
 	if err != nil {
 		return err
@@ -84,7 +82,7 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
 		return err
 	}
 
-	os.Chmod(p1, os.FileMode(rand.Intn(0o777)|0o400))
+	os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
 
 	t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
 	return os.Chtimes(p1, t, t)
@@ -107,11 +105,11 @@ func readRand(bs []byte) (int, error) {
 	return len(bs), nil
 }
 
-type infiniteReader struct {
+type inifiteReader struct {
 	rd io.ReadSeeker
 }
 
-func (i *infiniteReader) Read(bs []byte) (int, error) {
+func (i *inifiteReader) Read(bs []byte) (int, error) {
 	n, err := i.rd.Read(bs)
 	if err == io.EOF {
 		err = nil

cmd/strelaypoolsrv/README.md

@@ -21,3 +21,4 @@ See `relaypoolsrv -help` for configuration options.
 
 [oschwald/geoip2-golang](https://github.com/oschwald/geoip2-golang), [oschwald/maxminddb-golang](https://github.com/oschwald/maxminddb-golang), Copyright (C) 2015 [Gregory J. Oschwald](mailto:oschwald@gmail.com).
 
+[lib/pq](https://github.com/lib/pq)</a>, Copyright (C) 2011-2013 'pq' Contributors Portions Copyright (C) 2011 Blake Mizerany.

cmd/strelaypoolsrv/auto/doc.go

@@ -4,7 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//go:generate go run ../../../../script/genassets.go -o gui.files.go ../gui
+//go:generate go run ../../../script/genassets.go -o gui.files.go ../gui
 
 // Package auto contains auto generated files for web assets.
 package auto

cmd/strelaypoolsrv/auto/noassets.go

@@ -4,8 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//go:build noassets
-// +build noassets
+//+build noassets
 
 package auto
 

cmd/strelaypoolsrv/gui/index.html

@@ -237,7 +237,7 @@
       uptimeSeconds: 0,
     };
     $scope.map = L.map('map').setView([40.90296, 1.90925], 2);
-    L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png',
+    L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',
     {
       attribution: 'Leaflet',
       maxZoom: 17
@@ -259,7 +259,7 @@
       return a.value > b.value ? 1 : -1;
     }
 
-    $http.get("/endpoint/full").then(function(response) {
+    $http.get("/endpoint").then(function(response) {
       $scope.relays = response.data.relays;
 
       angular.forEach($scope.relays, function(relay) {
@@ -338,7 +338,7 @@
         relay.showMarker = function() {
           relay.marker.openPopup();
         }
-
+        
         relay.hideMarker = function() {
           relay.marker.closePopup();
         }
@@ -347,7 +347,7 @@
 
     function addCircleToMap(relay) {
       console.log(relay.location.latitude)
-      L.circle([relay.location.latitude, relay.location.longitude],
+      L.circle([relay.location.latitude, relay.location.longitude], 
         {
           radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000,
           color: "FF0000",

cmd/strelaypoolsrv/main.go

@@ -3,12 +3,15 @@
 package main
 
 import (
+	"compress/gzip"
 	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
@@ -17,21 +20,21 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"sync/atomic"
 	"time"
 
-	lru "github.com/hashicorp/golang-lru/v2"
+	"github.com/syncthing/syncthing/lib/protocol"
+
+	"github.com/golang/groupcache/lru"
+	"github.com/oschwald/geoip2-golang"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-
-	"github.com/syncthing/syncthing/cmd/infra/strelaypoolsrv/auto"
+	"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
 	"github.com/syncthing/syncthing/lib/assets"
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
-	"github.com/syncthing/syncthing/lib/geoip"
-	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/relay/client"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/tlsutil"
+	"golang.org/x/time/rate"
 )
 
 type location struct {
@@ -50,10 +53,6 @@ type relay struct {
 	StatsRetrieved time.Time `json:"statsRetrieved"`
 }
 
-type relayShort struct {
-	URL string `json:"url"`
-}
-
 type stats struct {
 	StartTime          time.Time `json:"startTime"`
 	UptimeSeconds      int       `json:"uptimeSeconds"`
@@ -98,18 +97,30 @@ var (
 	testCert          tls.Certificate
 	knownRelaysFile   = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
 	listen            = ":80"
-	metricsListen     = ":8081"
 	dir               string
 	evictionTime      = time.Hour
 	debug             bool
+	getLRUSize        = 10 << 10
+	getLimitBurst     = 10
+	getLimitAvg       = 2
+	postLRUSize       = 1 << 10
+	postLimitBurst    = 2
+	postLimitAvg      = 2
+	getLimit          time.Duration
+	postLimit         time.Duration
 	permRelaysFile    string
 	ipHeader          string
+	geoipPath         string
 	proto             string
-	statsRefresh      = time.Minute
-	requestQueueLen   = 64
-	requestProcessors = 8
-	geoipLicenseKey   = os.Getenv("GEOIP_LICENSE_KEY")
-	geoipAccountID, _ = strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID"))
+	statsRefresh      = time.Minute / 2
+	requestQueueLen   = 10
+	requestProcessors = 1
+
+	getMut      = sync.NewMutex()
+	getLRUCache *lru.Cache
+
+	postMut      = sync.NewMutex()
+	postLRUCache *lru.Cache
 
 	requests chan request
 
@@ -117,7 +128,6 @@ var (
 	knownRelays     = make([]*relay, 0)
 	permanentRelays = make([]*relay, 0)
 	evictionTimers  = make(map[string]*time.Timer)
-	globalBlocklist = newErrorTracker(1000)
 )
 
 const (
@@ -129,45 +139,51 @@ func main() {
 	log.SetFlags(log.Lshortfile)
 
 	flag.StringVar(&listen, "listen", listen, "Listen address")
-	flag.StringVar(&metricsListen, "metrics-listen", metricsListen, "Metrics listen address")
 	flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
 	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
 	flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
+	flag.IntVar(&getLRUSize, "get-limit-cache", getLRUSize, "Get request limiter cache size")
+	flag.IntVar(&getLimitAvg, "get-limit-avg", getLimitAvg, "Allowed average get request rate, per 10 s")
+	flag.IntVar(&getLimitBurst, "get-limit-burst", getLimitBurst, "Allowed burst get requests")
+	flag.IntVar(&postLRUSize, "post-limit-cache", postLRUSize, "Post request limiter cache size")
+	flag.IntVar(&postLimitAvg, "post-limit-avg", postLimitAvg, "Allowed average post request rate, per minute")
+	flag.IntVar(&postLimitBurst, "post-limit-burst", postLimitBurst, "Allowed burst post requests")
 	flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
-	flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays")
 	flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
+	flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
 	flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
 	flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
 	flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
 	flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
-	flag.StringVar(&geoipLicenseKey, "geoip-license-key", geoipLicenseKey, "License key for GeoIP database")
 
 	flag.Parse()
 
 	requests = make(chan request, requestQueueLen)
-	geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), geoipAccountID, geoipLicenseKey, os.TempDir())
-	if err != nil {
-		log.Fatalln("Failed to create GeoIP provider:", err)
-	}
-	go geoip.Serve(context.TODO())
+
+	getLimit = 10 * time.Second / time.Duration(getLimitAvg)
+	postLimit = time.Minute / time.Duration(postLimitAvg)
+
+	getLRUCache = lru.New(getLRUSize)
+	postLRUCache = lru.New(postLRUSize)
 
 	var listener net.Listener
+	var err error
 
 	if permRelaysFile != "" {
-		permanentRelays = loadRelays(permRelaysFile, geoip)
+		permanentRelays = loadRelays(permRelaysFile)
 	}
 
 	testCert = createTestCertificate()
 
 	for i := 0; i < requestProcessors; i++ {
-		go requestProcessor(geoip)
+		go requestProcessor()
 	}
 
 	// Load relays from cache in the background.
 	// Load them in a serial fashion to make sure any genuine requests
 	// are not dropped.
 	go func() {
-		for _, relay := range loadRelays(knownRelaysFile, geoip) {
+		for _, relay := range loadRelays(knownRelaysFile) {
 			resultChan := make(chan result)
 			requests <- request{relay, resultChan, nil}
 			result := <-resultChan
@@ -223,40 +239,15 @@ func main() {
 		log.Fatalln("listen:", err)
 	}
 
-	if metricsListen != "" {
-		mmux := http.NewServeMux()
-		mmux.HandleFunc("/metrics", handleMetrics)
-		go func() {
-			if err := http.ListenAndServe(metricsListen, mmux); err != nil {
-				log.Fatalln("HTTP serve metrics:", err)
-			}
-		}()
-	}
-
-	getMux := http.NewServeMux()
-	getMux.HandleFunc("/", handleAssets)
-	getMux.HandleFunc("/endpoint", withAPIMetrics(handleEndpointShort))
-	getMux.HandleFunc("/endpoint/full", withAPIMetrics(handleEndpointFull))
-
-	postMux := http.NewServeMux()
-	postMux.HandleFunc("/endpoint", withAPIMetrics(handleRegister))
-
-	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		switch r.Method {
-		case http.MethodGet, http.MethodHead, http.MethodOptions:
-			getMux.ServeHTTP(w, r)
-		case http.MethodPost:
-			postMux.ServeHTTP(w, r)
-		default:
-			http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
-		}
-	})
+	handler := http.NewServeMux()
+	handler.HandleFunc("/", handleAssets)
+	handler.HandleFunc("/endpoint", handleRequest)
+	handler.HandleFunc("/metrics", handleMetrics)
 
 	srv := http.Server{
 		Handler:     handler,
 		ReadTimeout: 10 * time.Second,
 	}
-	srv.SetKeepAlivesEnabled(false)
 
 	err = srv.Serve(listener)
 	if err != nil {
@@ -290,24 +281,43 @@ func handleAssets(w http.ResponseWriter, r *http.Request) {
 	assets.Serve(w, r, as)
 }
 
-func withAPIMetrics(next http.HandlerFunc) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
-		w = NewLoggingResponseWriter(w)
-		defer func() {
-			timer.ObserveDuration()
-			lw := w.(*loggingResponseWriter)
-			apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
-		}()
-		next(w, r)
+func handleRequest(w http.ResponseWriter, r *http.Request) {
+	timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
+
+	w = NewLoggingResponseWriter(w)
+	defer func() {
+		timer.ObserveDuration()
+		lw := w.(*loggingResponseWriter)
+		apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
+	}()
+
+	if ipHeader != "" {
+		r.RemoteAddr = r.Header.Get(ipHeader)
+	}
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	switch r.Method {
+	case "GET":
+		if limit(r.RemoteAddr, getLRUCache, getMut, getLimit, getLimitBurst) {
+			w.WriteHeader(httpStatusEnhanceYourCalm)
+			return
+		}
+		handleGetRequest(w, r)
+	case "POST":
+		if limit(r.RemoteAddr, postLRUCache, postMut, postLimit, postLimitBurst) {
+			w.WriteHeader(httpStatusEnhanceYourCalm)
+			return
+		}
+		handlePostRequest(w, r)
+	default:
+		if debug {
+			log.Println("Unhandled HTTP method", r.Method)
+		}
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 	}
 }
 
-// handleEndpointFull returns the relay list with full metadata and
-// statistics. Large, and expensive.
-func handleEndpointFull(rw http.ResponseWriter, r *http.Request) {
+func handleGetRequest(rw http.ResponseWriter, r *http.Request) {
 	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
-	rw.Header().Set("Access-Control-Allow-Origin", "*")
 
 	mut.RLock()
 	relays := make([]*relay, len(permanentRelays)+len(knownRelays))
@@ -315,52 +325,23 @@ func handleEndpointFull(rw http.ResponseWriter, r *http.Request) {
 	copy(relays[n:], knownRelays)
 	mut.RUnlock()
 
-	_ = json.NewEncoder(rw).Encode(map[string][]*relay{
+	// Shuffle
+	rand.Shuffle(relays)
+
+	w := io.Writer(rw)
+	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+		rw.Header().Set("Content-Encoding", "gzip")
+		gw := gzip.NewWriter(rw)
+		defer gw.Close()
+		w = gw
+	}
+
+	_ = json.NewEncoder(w).Encode(map[string][]*relay{
 		"relays": relays,
 	})
 }
 
-// handleEndpointShort returns the relay list with only the URL.
-func handleEndpointShort(rw http.ResponseWriter, r *http.Request) {
-	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
-	rw.Header().Set("Access-Control-Allow-Origin", "*")
-
-	mut.RLock()
-	relays := make([]relayShort, 0, len(permanentRelays)+len(knownRelays))
-	for _, r := range append(permanentRelays, knownRelays...) {
-		relays = append(relays, relayShort{URL: slimURL(r.URL)})
-	}
-	mut.RUnlock()
-
-	_ = json.NewEncoder(rw).Encode(map[string][]relayShort{
-		"relays": relays,
-	})
-}
-
-func handleRegister(w http.ResponseWriter, r *http.Request) {
-	// Get the IP address of the client
-	rhost := r.RemoteAddr
-	if ipHeader != "" {
-		hdr := r.Header.Get(ipHeader)
-		fields := strings.Split(hdr, ",")
-		if len(fields) > 0 {
-			rhost = strings.TrimSpace(fields[len(fields)-1])
-		}
-	}
-	if host, _, err := net.SplitHostPort(rhost); err == nil {
-		rhost = host
-	}
-
-	// Check the black list. A client is blacklisted if their last 10
-	// attempts to join have all failed. The "Unauthorized" status return
-	// causes strelaysrv to cease attempting to join.
-	if globalBlocklist.IsBlocked(rhost) {
-		log.Println("Rejected blocked client", rhost)
-		http.Error(w, "Too many errors", http.StatusUnauthorized)
-		globalBlocklist.ClearErrors(rhost)
-		return
-	}
-
+func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 	var relayCert *x509.Certificate
 	if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
 		relayCert = r.TLS.PeerCertificates[0]
@@ -388,11 +369,6 @@ func handleRegister(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Canonicalize the URL. In particular, parse and re-encode the query
-	// string so that it's guaranteed to be valid.
-	uri.RawQuery = uri.Query().Encode()
-	newRelay.URL = uri.String()
-
 	if relayCert != nil {
 		advertisedId := uri.Query().Get("id")
 		idFromCert := protocol.NewDeviceID(relayCert.Raw).String()
@@ -412,6 +388,12 @@ func handleRegister(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Get the IP address of the client
+	rhost := r.RemoteAddr
+	if host, _, err := net.SplitHostPort(rhost); err == nil {
+		rhost = host
+	}
+
 	ip := net.ParseIP(host)
 	// The client did not provide an IP address, use the IP address of the client.
 	if ip == nil || ip.IsUnspecified() {
@@ -443,14 +425,10 @@ func handleRegister(w http.ResponseWriter, r *http.Request) {
 	case requests <- request{&newRelay, reschan, prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("queue"))}:
 		result := <-reschan
 		if result.err != nil {
-			log.Println("Join from", r.RemoteAddr, "failed:", result.err)
-			globalBlocklist.AddError(rhost)
 			relayTestsTotal.WithLabelValues("failed").Inc()
 			http.Error(w, result.err.Error(), http.StatusBadRequest)
 			return
 		}
-		log.Println("Join from", r.RemoteAddr, "succeeded")
-		globalBlocklist.ClearErrors(rhost)
 		relayTestsTotal.WithLabelValues("success").Inc()
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		json.NewEncoder(w).Encode(map[string]time.Duration{
@@ -466,19 +444,19 @@ func handleRegister(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func requestProcessor(geoip *geoip.Provider) {
+func requestProcessor() {
 	for request := range requests {
 		if request.queueTimer != nil {
 			request.queueTimer.ObserveDuration()
 		}
 
 		timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test"))
-		handleRelayTest(request, geoip)
+		handleRelayTest(request)
 		timer.ObserveDuration()
 	}
 }
 
-func handleRelayTest(request request, geoip *geoip.Provider) {
+func handleRelayTest(request request) {
 	if debug {
 		log.Println("Request for", request.relay)
 	}
@@ -491,7 +469,7 @@ func handleRelayTest(request request, geoip *geoip.Provider) {
 	}
 
 	stats := fetchStats(request.relay)
-	location := getLocation(request.relay.uri.Host, geoip)
+	location := getLocation(request.relay.uri.Host)
 
 	mut.Lock()
 	if stats != nil {
@@ -564,8 +542,25 @@ func evict(relay *relay) func() {
 	}
 }
 
-func loadRelays(file string, geoip *geoip.Provider) []*relay {
-	content, err := os.ReadFile(file)
+func limit(addr string, cache *lru.Cache, lock sync.Mutex, intv time.Duration, burst int) bool {
+	if host, _, err := net.SplitHostPort(addr); err == nil {
+		addr = host
+	}
+
+	lock.Lock()
+	v, _ := cache.Get(addr)
+	bkt, ok := v.(*rate.Limiter)
+	if !ok {
+		bkt = rate.NewLimiter(rate.Every(intv), burst)
+		cache.Add(addr, bkt)
+	}
+	lock.Unlock()
+
+	return !bkt.Allow()
+}
+
+func loadRelays(file string) []*relay {
+	content, err := ioutil.ReadFile(file)
 	if err != nil {
 		log.Println("Failed to load relays: " + err.Error())
 		return nil
@@ -573,7 +568,7 @@ func loadRelays(file string, geoip *geoip.Provider) []*relay {
 
 	var relays []*relay
 	for _, line := range strings.Split(string(content), "\n") {
-		if line == "" {
+		if len(line) == 0 {
 			continue
 		}
 
@@ -588,7 +583,7 @@ func loadRelays(file string, geoip *geoip.Provider) []*relay {
 
 		relays = append(relays, &relay{
 			URL:      line,
-			Location: getLocation(uri.Host, geoip),
+			Location: getLocation(uri.Host),
 			uri:      uri,
 		})
 		if debug {
@@ -603,11 +598,11 @@ func saveRelays(file string, relays []*relay) error {
 	for _, relay := range relays {
 		content += relay.uri.String() + "\n"
 	}
-	return os.WriteFile(file, []byte(content), 0o777)
+	return ioutil.WriteFile(file, []byte(content), 0777)
 }
 
 func createTestCertificate() tls.Certificate {
-	tmpDir, err := os.MkdirTemp("", "relaypoolsrv")
+	tmpDir, err := ioutil.TempDir("", "relaypoolsrv")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -621,16 +616,21 @@ func createTestCertificate() tls.Certificate {
 	return cert
 }
 
-func getLocation(host string, geoip *geoip.Provider) location {
+func getLocation(host string) location {
 	timer := prometheus.NewTimer(locationLookupSeconds)
 	defer timer.ObserveDuration()
+	db, err := geoip2.Open(geoipPath)
+	if err != nil {
+		return location{}
+	}
+	defer db.Close()
 
 	addr, err := net.ResolveTCPAddr("tcp", host)
 	if err != nil {
 		return location{}
 	}
 
-	city, err := geoip.City(addr.IP)
+	city, err := db.City(addr.IP)
 	if err != nil {
 		return location{}
 	}
@@ -657,55 +657,3 @@ func (lrw *loggingResponseWriter) WriteHeader(code int) {
 	lrw.statusCode = code
 	lrw.ResponseWriter.WriteHeader(code)
 }
-
-type errorTracker struct {
-	errors *lru.TwoQueueCache[string, *errorCounter]
-}
-
-type errorCounter struct {
-	count atomic.Int32
-}
-
-func newErrorTracker(size int) *errorTracker {
-	cache, err := lru.New2Q[string, *errorCounter](size)
-	if err != nil {
-		panic(err)
-	}
-	return &errorTracker{
-		errors: cache,
-	}
-}
-
-func (b *errorTracker) AddError(host string) {
-	entry, ok := b.errors.Get(host)
-	if !ok {
-		entry = &errorCounter{}
-		b.errors.Add(host, entry)
-	}
-	c := entry.count.Add(1)
-	log.Printf("Error count for %s is now %d", host, c)
-}
-
-func (b *errorTracker) ClearErrors(host string) {
-	b.errors.Remove(host)
-}
-
-func (b *errorTracker) IsBlocked(host string) bool {
-	if be, ok := b.errors.Get(host); ok {
-		return be.count.Load() > 10
-	}
-	return false
-}
-
-func slimURL(u string) string {
-	p, err := url.Parse(u)
-	if err != nil {
-		return u
-	}
-	newQuery := url.Values{}
-	if id := p.Query().Get("id"); id != "" {
-		newQuery.Set("id", id)
-	}
-	p.RawQuery = newQuery.Encode()
-	return p.String()
-}

cmd/strelaypoolsrv/main_test.go

@@ -11,7 +11,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http/httptest"
-	"net/url"
 	"strings"
 	"sync"
 	"testing"
@@ -42,7 +41,7 @@ func TestHandleGetRequest(t *testing.T) {
 
 	w := httptest.NewRecorder()
 	w.Body = new(bytes.Buffer)
-	handleEndpointFull(w, httptest.NewRequest("GET", "/", nil))
+	handleGetRequest(w, httptest.NewRequest("GET", "/", nil))
 
 	result := make(map[string][]*relay)
 	err := json.NewDecoder(w.Body).Decode(&result)
@@ -66,44 +65,3 @@ func TestHandleGetRequest(t *testing.T) {
 		}
 	}
 }
-
-func TestCanonicalizeQueryValues(t *testing.T) {
-	// This just demonstrates and validates the uri.Parse/String stuff in
-	// regards to query strings.
-
-	in := "http://example.com/?some weird= query^value"
-	exp := "http://example.com/?some+weird=+query%5Evalue"
-
-	uri, err := url.Parse(in)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	str := uri.String()
-	if str != in {
-		// Just re-encoding the URL doesn't sanitize the query string.
-		t.Errorf("expected %q, got %q", in, str)
-	}
-
-	uri.RawQuery = uri.Query().Encode()
-	str = uri.String()
-	if str != exp {
-		// The query string is now in correct format.
-		t.Errorf("expected %q, got %q", exp, str)
-	}
-}
-
-func TestSlimURL(t *testing.T) {
-	cases := []struct {
-		in, out string
-	}{
-		{"http://example.com/", "http://example.com/"},
-		{"relay://192.0.2.42:22067/?globalLimitBps=0&id=EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M&networkTimeout=2m0s&pingInterval=1m0s&providedBy=Test&sessionLimitBps=0&statusAddr=%3A22070", "relay://192.0.2.42:22067/?id=EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M-EIC6B3M"},
-	}
-
-	for _, c := range cases {
-		if got := slimURL(c.in); got != c.out {
-			t.Errorf("expected %q, got %q", c.out, got)
-		}
-	}
-}

cmd/strelaypoolsrv/stats.go

@@ -6,12 +6,26 @@ import (
 	"encoding/json"
 	"net"
 	"net/http"
+	"os"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
+func init() {
+	processCollectorOpts := prometheus.ProcessCollectorOpts{
+		Namespace: "syncthing_relaypoolsrv",
+		PidFn: func() (int, error) {
+			return os.Getpid(), nil
+		},
+	}
+
+	prometheus.MustRegister(
+		prometheus.NewProcessCollector(processCollectorOpts),
+	)
+}
+
 var (
 	statusClient = http.Client{
 		Timeout: 5 * time.Second,

cmd/strelaysrv/listener.go

@@ -12,17 +12,18 @@ import (
 	"time"
 
 	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/relay/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
+
+	"github.com/syncthing/syncthing/lib/relay/protocol"
 )
 
 var (
 	outboxesMut    = sync.RWMutex{}
 	outboxes       = make(map[syncthingprotocol.DeviceID]chan interface{})
-	numConnections atomic.Int64
+	numConnections int64
 )
 
-func listener(_, addr string, config *tls.Config, token string) {
+func listener(proto, addr string, config *tls.Config) {
 	tcpListener, err := net.Listen("tcp", addr)
 	if err != nil {
 		log.Fatalln(err)
@@ -35,14 +36,8 @@ func listener(_, addr string, config *tls.Config, token string) {
 	for {
 		conn, isTLS, err := listener.AcceptNoWrapTLS()
 		if err != nil {
-			// Conn may be nil if accept failed, or non-nil if the initial
-			// read to figure out if it's TLS or not failed. In the latter
-			// case, close the connection before moving on.
-			if conn != nil {
-				conn.Close()
-			}
 			if debug {
-				log.Println("Listener failed to accept:", err)
+				log.Println("Listener failed to accept connection from", conn.RemoteAddr(), ". Possibly a TCP Ping.")
 			}
 			continue
 		}
@@ -54,7 +49,7 @@ func listener(_, addr string, config *tls.Config, token string) {
 		}
 
 		if isTLS {
-			go protocolConnectionHandler(conn, config, token)
+			go protocolConnectionHandler(conn, config)
 		} else {
 			go sessionConnectionHandler(conn)
 		}
@@ -62,7 +57,7 @@ func listener(_, addr string, config *tls.Config, token string) {
 	}
 }
 
-func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token string) {
+func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
 	conn := tls.Server(tcpConn, config)
 	if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil {
 		if debug {
@@ -81,7 +76,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token strin
 	}
 
 	state := conn.ConnectionState()
-	if debug && state.NegotiatedProtocol != protocol.ProtocolName {
+	if (!state.NegotiatedProtocolIsMutual || state.NegotiatedProtocol != protocol.ProtocolName) && debug {
 		log.Println("Protocol negotiation error")
 	}
 
@@ -124,16 +119,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token strin
 
 			switch msg := message.(type) {
 			case protocol.JoinRelayRequest:
-				if token != "" && msg.Token != token {
-					if debug {
-						log.Printf("invalid token %s\n", msg.Token)
-					}
-					protocol.WriteMessage(conn, protocol.ResponseWrongToken)
-					conn.Close()
-					continue
-				}
-
-				if overLimit.Load() {
+				if atomic.LoadInt32(&overLimit) > 0 {
 					protocol.WriteMessage(conn, protocol.RelayFull{})
 					if debug {
 						log.Println("Refusing join request from", id, "due to being over limits")
@@ -272,7 +258,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token strin
 				conn.Close()
 			}
 
-			if overLimit.Load() && !hasSessions(id) {
+			if atomic.LoadInt32(&overLimit) > 0 && !hasSessions(id) {
 				if debug {
 					log.Println("Dropping", id, "as it has no sessions and we are over our limits")
 				}
@@ -365,8 +351,8 @@ func sessionConnectionHandler(conn net.Conn) {
 }
 
 func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) {
-	numConnections.Add(1)
-	defer numConnections.Add(-1)
+	atomic.AddInt64(&numConnections, 1)
+	defer atomic.AddInt64(&numConnections, -1)
 
 	for {
 		msg, err := protocol.ReadMessage(conn)

cmd/strelaysrv/main.go

@@ -14,24 +14,25 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"sync/atomic"
 	"syscall"
 	"time"
 
-	"golang.org/x/time/rate"
-
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/build"
-	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/events"
-	"github.com/syncthing/syncthing/lib/nat"
 	"github.com/syncthing/syncthing/lib/osutil"
-	_ "github.com/syncthing/syncthing/lib/pmp"
-	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/relay/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
+	"golang.org/x/time/rate"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/nat"
+	_ "github.com/syncthing/syncthing/lib/pmp"
 	_ "github.com/syncthing/syncthing/lib/upnp"
+
+	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
 )
 
 var (
@@ -49,14 +50,13 @@ var (
 
 	sessionLimitBps   int
 	globalLimitBps    int
-	overLimit         atomic.Bool
+	overLimit         int32
 	descriptorLimit   int64
 	sessionLimiter    *rate.Limiter
 	globalLimiter     *rate.Limiter
 	networkBufferSize int
 
 	statusAddr       string
-	token            string
 	poolAddrs        string
 	pools            []string
 	providedBy       string
@@ -90,7 +90,6 @@ func main() {
 	flag.IntVar(&globalLimitBps, "global-rate", globalLimitBps, "Global rate limit, in bytes/s")
 	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
 	flag.StringVar(&statusAddr, "status-srv", ":22070", "Listen address for status service (blank to disable)")
-	flag.StringVar(&token, "token", "", "Token to restrict access to the relay (optional). Disables joining any pools.")
 	flag.StringVar(&poolAddrs, "pools", defaultPoolAddrs, "Comma separated list of relay pool addresses to join")
 	flag.StringVar(&providedBy, "provided-by", "", "An optional description about who provides the relay")
 	flag.StringVar(&extAddress, "ext-address", "", "An optional address to advertise as being available on.\n\tAllows listening on an unprivileged port with port forwarding from e.g. 443, and be connected to on port 443.")
@@ -147,7 +146,7 @@ func main() {
 		log.Println("Connection limit", descriptorLimit)
 
 		go monitorLimits()
-	} else if err != nil && !build.IsWindows {
+	} else if err != nil && runtime.GOOS != "windows" {
 		log.Println("Assuming no connection limit, due to error retrieving rlimits:", err)
 	}
 
@@ -194,22 +193,14 @@ func main() {
 		cfg.Options.NATTimeoutS = natTimeout
 	})
 	natSvc := nat.NewService(id, wrapper)
-	var ipVersion nat.IPVersion
-	if strings.HasSuffix(proto, "4") {
-		ipVersion = nat.IPv4Only
-	} else if strings.HasSuffix(proto, "6") {
-		ipVersion = nat.IPv6Only
-	} else {
-		ipVersion = nat.IPvAny
-	}
-	mapping := mapping{natSvc.NewMapping(nat.TCP, ipVersion, addr.IP, addr.Port)}
+	mapping := mapping{natSvc.NewMapping(nat.TCP, addr.IP, addr.Port)}
 
 	if natEnabled {
 		ctx, cancel := context.WithCancel(context.Background())
 		go natSvc.Serve(ctx)
 		defer cancel()
 		found := make(chan struct{})
-		mapping.OnChanged(func() {
+		mapping.OnChanged(func(_ *nat.Mapping, _, _ []nat.Address) {
 			select {
 			case found <- struct{}{}:
 			default:
@@ -239,37 +230,14 @@ func main() {
 		go statusService(statusAddr)
 	}
 
-	uri, err := url.Parse(fmt.Sprintf("relay://%s/", mapping.Address()))
+	uri, err := url.Parse(fmt.Sprintf("relay://%s/?id=%s&pingInterval=%s&networkTimeout=%s&sessionLimitBps=%d&globalLimitBps=%d&statusAddr=%s&providedBy=%s", mapping.Address(), id, pingInterval, networkTimeout, sessionLimitBps, globalLimitBps, statusAddr, providedBy))
 	if err != nil {
 		log.Fatalln("Failed to construct URI", err)
 		return
 	}
 
-	// Add properly encoded query string parameters to URL.
-	query := make(url.Values)
-	query.Set("id", id.String())
-	query.Set("pingInterval", pingInterval.String())
-	query.Set("networkTimeout", networkTimeout.String())
-	if sessionLimitBps > 0 {
-		query.Set("sessionLimitBps", fmt.Sprint(sessionLimitBps))
-	}
-	if globalLimitBps > 0 {
-		query.Set("globalLimitBps", fmt.Sprint(globalLimitBps))
-	}
-	if statusAddr != "" {
-		query.Set("statusAddr", statusAddr)
-	}
-	if providedBy != "" {
-		query.Set("providedBy", providedBy)
-	}
-	uri.RawQuery = query.Encode()
-
 	log.Println("URI:", uri.String())
 
-	if token != "" {
-		poolAddrs = ""
-	}
-
 	if poolAddrs == defaultPoolAddrs {
 		log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
 		log.Println("!!  Joining default relay pools, this relay will be available for public use. !!")
@@ -285,7 +253,7 @@ func main() {
 		}
 	}
 
-	go listener(proto, listen, tlsCfg, token)
+	go listener(proto, listen, tlsCfg)
 
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
@@ -316,10 +284,10 @@ func main() {
 func monitorLimits() {
 	limitCheckTimer = time.NewTimer(time.Minute)
 	for range limitCheckTimer.C {
-		if numConnections.Load()+numProxies.Load() > descriptorLimit {
-			overLimit.Store(true)
+		if atomic.LoadInt64(&numConnections)+atomic.LoadInt64(&numProxies) > descriptorLimit {
+			atomic.StoreInt32(&overLimit, 1)
 			log.Println("Gone past our connection limits. Starting to refuse new/drop idle connections.")
-		} else if overLimit.CompareAndSwap(true, false) {
+		} else if atomic.CompareAndSwapInt32(&overLimit, 1, 0) {
 			log.Println("Dropped below our connection limits. Accepting new connections.")
 		}
 		limitCheckTimer.Reset(time.Minute)

cmd/strelaysrv/pool.go

@@ -6,7 +6,7 @@ import (
 	"bytes"
 	"crypto/tls"
 	"encoding/json"
-	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"net/url"
@@ -56,7 +56,7 @@ func poolHandler(pool string, uri *url.URL, mapping mapping, ownCert tls.Certifi
 			continue
 		}
 
-		bs, err := io.ReadAll(resp.Body)
+		bs, err := ioutil.ReadAll(resp.Body)
 		resp.Body.Close()
 		if err != nil {
 			log.Printf("Error joining pool %v: reading response: %v", pool, err)
@@ -74,8 +74,9 @@ func poolHandler(pool string, uri *url.URL, mapping mapping, ownCert tls.Certifi
 				log.Printf("Joined pool %s, rejoining in %v", pool, rejoin)
 				time.Sleep(rejoin)
 				continue
+			} else {
+				log.Printf("Joined pool %s, failed to deserialize response: %v", pool, err)
 			}
-			log.Printf("Joined pool %s, failed to deserialize response: %v", pool, err)
 
 		case http.StatusInternalServerError:
 			log.Printf("Failed to join %v: server error", pool)

cmd/strelaysrv/session.go

@@ -23,8 +23,8 @@ var (
 	sessionMut      = sync.RWMutex{}
 	activeSessions  = make([]*session, 0)
 	pendingSessions = make(map[string]*session)
-	numProxies      atomic.Int64
-	bytesProxied    atomic.Int64
+	numProxies      int64
+	bytesProxied    int64
 )
 
 func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionRateLimit, globalRateLimit *rate.Limiter) *session {
@@ -68,6 +68,7 @@ func findSession(key string) *session {
 	ses, ok := pendingSessions[key]
 	if !ok {
 		return nil
+
 	}
 	delete(pendingSessions, key)
 	return ses
@@ -250,8 +251,8 @@ func (s *session) proxy(c1, c2 net.Conn) error {
 		log.Println("Proxy", c1.RemoteAddr(), "->", c2.RemoteAddr())
 	}
 
-	numProxies.Add(1)
-	defer numProxies.Add(-1)
+	atomic.AddInt64(&numProxies, 1)
+	defer atomic.AddInt64(&numProxies, -1)
 
 	buf := make([]byte, networkBufferSize)
 	for {
@@ -261,7 +262,7 @@ func (s *session) proxy(c1, c2 net.Conn) error {
 			return err
 		}
 
-		bytesProxied.Add(int64(n))
+		atomic.AddInt64(&bytesProxied, int64(n))
 
 		if debug {
 			log.Printf("%d bytes from %s to %s", n, c1.RemoteAddr(), c2.RemoteAddr())

cmd/strelaysrv/status.go

@@ -36,7 +36,7 @@ func statusService(addr string) {
 	}
 }
 
-func getStatus(w http.ResponseWriter, _ *http.Request) {
+func getStatus(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Access-Control-Allow-Origin", "*")
 	status := make(map[string]interface{})
 
@@ -51,9 +51,9 @@ func getStatus(w http.ResponseWriter, _ *http.Request) {
 	status["numPendingSessionKeys"] = len(pendingSessions)
 	status["numActiveSessions"] = len(activeSessions)
 	sessionMut.Unlock()
-	status["numConnections"] = numConnections.Load()
-	status["numProxies"] = numProxies.Load()
-	status["bytesProxied"] = bytesProxied.Load()
+	status["numConnections"] = atomic.LoadInt64(&numConnections)
+	status["numProxies"] = atomic.LoadInt64(&numProxies)
+	status["bytesProxied"] = atomic.LoadInt64(&bytesProxied)
 	status["goVersion"] = runtime.Version()
 	status["goOS"] = runtime.GOOS
 	status["goArch"] = runtime.GOARCH
@@ -88,13 +88,13 @@ func getStatus(w http.ResponseWriter, _ *http.Request) {
 }
 
 type rateCalculator struct {
-	counter   *atomic.Int64
+	counter   *int64 // atomic, must remain 64-bit aligned
 	rates     []int64
 	prev      int64
 	startTime time.Time
 }
 
-func newRateCalculator(keepIntervals int, interval time.Duration, counter *atomic.Int64) *rateCalculator {
+func newRateCalculator(keepIntervals int, interval time.Duration, counter *int64) *rateCalculator {
 	r := &rateCalculator{
 		rates:     make([]int64, keepIntervals),
 		counter:   counter,
@@ -112,7 +112,7 @@ func (r *rateCalculator) updateRates(interval time.Duration) {
 		next := now.Truncate(interval).Add(interval)
 		time.Sleep(next.Sub(now))
 
-		cur := r.counter.Load()
+		cur := atomic.LoadInt64(r.counter)
 		rate := int64(float64(cur-r.prev) / interval.Seconds())
 		copy(r.rates[1:], r.rates)
 		r.rates[0] = rate

cmd/strelaysrv/testutil/main.go

@@ -14,7 +14,6 @@ import (
 	"path/filepath"
 	"time"
 
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/relay/client"
 	"github.com/syncthing/syncthing/lib/relay/protocol"
@@ -128,6 +127,10 @@ func stdinReader(c chan<- string) {
 }
 
 func connectToStdio(stdin <-chan string, conn net.Conn) {
+	go func() {
+
+	}()
+
 	buf := make([]byte, 1024)
 	for {
 		conn.SetReadDeadline(time.Now().Add(time.Millisecond))

cmd/stsigtool/main.go

@@ -9,10 +9,10 @@ package main
 import (
 	"flag"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/signature"
 	"github.com/syncthing/syncthing/lib/upgrade"
 )
@@ -69,7 +69,7 @@ func gen() {
 }
 
 func sign(keyname, dataname string) {
-	privkey, err := os.ReadFile(keyname)
+	privkey, err := ioutil.ReadFile(keyname)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -95,7 +95,7 @@ func sign(keyname, dataname string) {
 }
 
 func verifyWithFile(signame, dataname, keyname string) {
-	pubkey, err := os.ReadFile(keyname)
+	pubkey, err := ioutil.ReadFile(keyname)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -103,7 +103,7 @@ func verifyWithFile(signame, dataname, keyname string) {
 }
 
 func verifyWithKey(signame, dataname string, pubkey []byte) {
-	sig, err := os.ReadFile(signame)
+	sig, err := ioutil.ReadFile(signame)
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/stupgrades/main.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"os"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/upgrade"
+)
+
+const defaultURL = "https://api.github.com/repos/syncthing/syncthing/releases?per_page=25"
+
+func main() {
+	url := flag.String("u", defaultURL, "GitHub releases url")
+	flag.Parse()
+
+	rels := upgrade.FetchLatestReleases(*url, "")
+	if rels == nil {
+		// An error was already logged
+		os.Exit(1)
+	}
+
+	sort.Sort(upgrade.SortByRelease(rels))
+	rels = filterForLatest(rels)
+
+	if err := json.NewEncoder(os.Stdout).Encode(rels); err != nil {
+		os.Exit(1)
+	}
+}
+
+// filterForLatest returns the latest stable and prerelease only. If the
+// stable version is newer (comes first in the list) there is no need to go
+// looking for a prerelease at all.
+func filterForLatest(rels []upgrade.Release) []upgrade.Release {
+	var filtered []upgrade.Release
+	var havePre bool
+	for _, rel := range rels {
+		if !rel.Prerelease {
+			// We found a stable version, we're good now.
+			filtered = append(filtered, rel)
+			break
+		}
+		if rel.Prerelease && !havePre {
+			// We remember the first prerelease we find.
+			filtered = append(filtered, rel)
+			havePre = true
+		}
+	}
+	return filtered
+}

cmd/stvanity/main.go

@@ -26,7 +26,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -45,7 +44,7 @@ func main() {
 
 	found := make(chan result)
 	stop := make(chan struct{})
-	var count atomic.Int64
+	var count int64
 
 	// Print periodic progress reports.
 	go printProgress(prefix, &count)
@@ -73,7 +72,7 @@ func main() {
 // Try certificates until one is found that has the prefix at the start of
 // the resulting device ID. Increments count atomically, sends the result to
 // found, returns when stop is closed.
-func generatePrefixed(prefix string, count *atomic.Int64, found chan<- result, stop <-chan struct{}) {
+func generatePrefixed(prefix string, count *int64, found chan<- result, stop <-chan struct{}) {
 	notBefore := time.Now()
 	notAfter := time.Date(2049, 12, 31, 23, 59, 59, 0, time.UTC)
 
@@ -110,7 +109,7 @@ func generatePrefixed(prefix string, count *atomic.Int64, found chan<- result, s
 		}
 
 		id := protocol.NewDeviceID(derBytes)
-		count.Add(1)
+		atomic.AddInt64(count, 1)
 
 		if strings.HasPrefix(id.String(), prefix) {
 			select {
@@ -122,7 +121,7 @@ func generatePrefixed(prefix string, count *atomic.Int64, found chan<- result, s
 	}
 }
 
-func printProgress(prefix string, count *atomic.Int64) {
+func printProgress(prefix string, count *int64) {
 	started := time.Now()
 	wantBits := 5 * len(prefix)
 	if wantBits > 63 {
@@ -133,7 +132,7 @@ func printProgress(prefix string, count *atomic.Int64) {
 	fmt.Printf("Want %d bits for prefix %q, about %.2g certs to test (statistically speaking)\n", wantBits, prefix, expectedIterations)
 
 	for range time.NewTicker(15 * time.Second).C {
-		tried := count.Load()
+		tried := atomic.LoadInt64(count)
 		elapsed := time.Since(started)
 		rate := float64(tried) / elapsed.Seconds()
 		expected := timeStr(expectedIterations / rate)
@@ -158,7 +157,7 @@ func saveCert(priv interface{}, derBytes []byte) {
 		os.Exit(1)
 	}
 
-	keyOut, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
+	keyOut, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)

cmd/stwatchfile/main.go

@@ -7,14 +7,13 @@
 package main
 
 import (
-	"crypto/sha256"
 	"flag"
 	"fmt"
 	"io"
 	"os"
 	"time"
 
-	_ "github.com/syncthing/syncthing/lib/automaxprocs"
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 func main() {

cmd/syncthing/cli/client.go

@@ -10,10 +10,8 @@ import (
 	"bytes"
 	"context"
 	"crypto/tls"
-	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"net"
 	"net/http"
 	"strings"
@@ -27,7 +25,6 @@ import (
 type APIClient interface {
 	Get(url string) (*http.Response, error)
 	Post(url, body string) (*http.Response, error)
-	PutJSON(url string, o interface{}) (*http.Response, error)
 }
 
 type apiClient struct {
@@ -121,36 +118,20 @@ func (c *apiClient) Do(req *http.Request) (*http.Response, error) {
 	return resp, checkResponse(resp)
 }
 
-func (c *apiClient) Request(url, method string, r io.Reader) (*http.Response, error) {
-	request, err := http.NewRequest(method, c.Endpoint()+"rest/"+url, r)
+func (c *apiClient) Get(url string) (*http.Response, error) {
+	request, err := http.NewRequest("GET", c.Endpoint()+"rest/"+url, nil)
 	if err != nil {
 		return nil, err
 	}
 	return c.Do(request)
 }
 
-func (c *apiClient) RequestString(url, method, data string) (*http.Response, error) {
-	return c.Request(url, method, bytes.NewBufferString(data))
-}
-
-func (c *apiClient) RequestJSON(url, method string, o interface{}) (*http.Response, error) {
-	data, err := json.Marshal(o)
+func (c *apiClient) Post(url, body string) (*http.Response, error) {
+	request, err := http.NewRequest("POST", c.Endpoint()+"rest/"+url, bytes.NewBufferString(body))
 	if err != nil {
 		return nil, err
 	}
-	return c.Request(url, method, bytes.NewBuffer(data))
-}
-
-func (c *apiClient) Get(url string) (*http.Response, error) {
-	return c.RequestString(url, "GET", "")
-}
-
-func (c *apiClient) Post(url, body string) (*http.Response, error) {
-	return c.RequestString(url, "POST", body)
-}
-
-func (c *apiClient) PutJSON(url string, o interface{}) (*http.Response, error) {
-	return c.RequestJSON(url, "PUT", o)
+	return c.Do(request)
 }
 
 var errNotFound = errors.New("invalid endpoint or API call")

cmd/syncthing/cli/config.go

@@ -8,12 +8,11 @@ package cli
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"reflect"
 
 	"github.com/AudriusButkevicius/recli"
-	"github.com/alecthomas/kong"
+	"github.com/pkg/errors"
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/urfave/cli"
 )
@@ -24,20 +23,9 @@ type configHandler struct {
 	err           error
 }
 
-type configCommand struct {
-	Args []string `arg:"" default:"-h"`
-}
-
-func (c *configCommand) Run(ctx Context, _ *kong.Context) error {
-	app := cli.NewApp()
-	app.Name = "syncthing"
-	app.Author = "The Syncthing Authors"
-	app.Metadata = map[string]interface{}{
-		"clientFactory": ctx.clientFactory,
-	}
-
+func getConfigCommand(f *apiClientFactory) (cli.Command, error) {
 	h := new(configHandler)
-	h.client, h.err = ctx.clientFactory.getClient()
+	h.client, h.err = f.getClient()
 	if h.err == nil {
 		h.cfg, h.err = getConfig(h.client)
 	}
@@ -50,15 +38,17 @@ func (c *configCommand) Run(ctx Context, _ *kong.Context) error {
 
 	commands, err := recli.New(recliCfg).Construct(&h.cfg)
 	if err != nil {
-		return fmt.Errorf("config reflect: %w", err)
+		return cli.Command{}, fmt.Errorf("config reflect: %w", err)
 	}
 
-	app.Commands = commands
-	app.HideHelp = true
-	app.Before = h.configBefore
-	app.After = h.configAfter
-
-	return app.Run(append([]string{app.Name}, c.Args...))
+	return cli.Command{
+		Name:        "config",
+		HideHelp:    true,
+		Usage:       "Configuration modification command group",
+		Subcommands: commands,
+		Before:      h.configBefore,
+		After:       h.configAfter,
+	}, nil
 }
 
 func (h *configHandler) configBefore(c *cli.Context) error {
@@ -70,7 +60,7 @@ func (h *configHandler) configBefore(c *cli.Context) error {
 	return h.err
 }
 
-func (h *configHandler) configAfter(_ *cli.Context) error {
+func (h *configHandler) configAfter(c *cli.Context) error {
 	if h.err != nil {
 		// Error was already returned in configBefore
 		return nil

cmd/syncthing/cli/debug.go

@@ -9,37 +9,47 @@ package cli
 import (
 	"fmt"
 	"net/url"
+
+	"github.com/urfave/cli"
 )
 
-type fileCommand struct {
-	FolderID string `arg:""`
-	Path     string `arg:""`
+var debugCommand = cli.Command{
+	Name:     "debug",
+	HideHelp: true,
+	Usage:    "Debug command group",
+	Subcommands: []cli.Command{
+		{
+			Name:      "file",
+			Usage:     "Show information about a file (or directory/symlink)",
+			ArgsUsage: "FOLDER-ID PATH",
+			Action:    expects(2, debugFile()),
+		},
+		indexCommand,
+		{
+			Name:      "profile",
+			Usage:     "Save a profile to help figuring out what Syncthing does.",
+			ArgsUsage: "cpu | heap",
+			Action:    expects(1, profile()),
+		},
+	},
 }
 
-func (f *fileCommand) Run(ctx Context) error {
-	indexDumpOutput := indexDumpOutputWrapper(ctx.clientFactory)
-
-	query := make(url.Values)
-	query.Set("folder", f.FolderID)
-	query.Set("file", normalizePath(f.Path))
-	return indexDumpOutput("debug/file?" + query.Encode())
-}
-
-type profileCommand struct {
-	Type string `arg:"" help:"cpu | heap"`
-}
-
-func (p *profileCommand) Run(ctx Context) error {
-	switch t := p.Type; t {
-	case "cpu", "heap":
-		return saveToFile(fmt.Sprintf("debug/%vprof", p.Type), ctx.clientFactory)
-	default:
-		return fmt.Errorf("expected cpu or heap as argument, got %v", t)
+func debugFile() cli.ActionFunc {
+	return func(c *cli.Context) error {
+		query := make(url.Values)
+		query.Set("folder", c.Args()[0])
+		query.Set("file", normalizePath(c.Args()[1]))
+		return indexDumpOutput("debug/file?" + query.Encode())(c)
 	}
 }
 
-type debugCommand struct {
-	File    fileCommand    `cmd:"" help:"Show information about a file (or directory/symlink)"`
-	Profile profileCommand `cmd:"" help:"Save a profile to help figuring out what Syncthing does"`
-	Index   indexCommand   `cmd:"" help:"Show information about the index (database)"`
+func profile() cli.ActionFunc {
+	return func(c *cli.Context) error {
+		switch t := c.Args()[0]; t {
+		case "cpu", "heap":
+			return saveToFile(fmt.Sprintf("debug/%vprof", c.Args()[0]))(c)
+		default:
+			return fmt.Errorf("expected cpu or heap as argument, got %v", t)
+		}
+	}
 }

cmd/syncthing/cli/errors.go

@@ -11,25 +11,36 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/alecthomas/kong"
+	"github.com/urfave/cli"
 )
 
-type errorsCommand struct {
-	Show  struct{}          `cmd:"" help:"Show pending errors"`
-	Push  errorsPushCommand `cmd:"" help:"Push an error to active clients"`
-	Clear struct{}          `cmd:"" help:"Clear pending errors"`
+var errorsCommand = cli.Command{
+	Name:     "errors",
+	HideHelp: true,
+	Usage:    "Error command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "show",
+			Usage:  "Show pending errors",
+			Action: expects(0, indexDumpOutput("system/error")),
+		},
+		{
+			Name:      "push",
+			Usage:     "Push an error to active clients",
+			ArgsUsage: "[error message]",
+			Action:    expects(1, errorsPush),
+		},
+		{
+			Name:   "clear",
+			Usage:  "Clear pending errors",
+			Action: expects(0, emptyPost("system/error/clear")),
+		},
+	},
 }
 
-type errorsPushCommand struct {
-	ErrorMessage string `arg:""`
-}
-
-func (e *errorsPushCommand) Run(ctx Context) error {
-	client, err := ctx.clientFactory.getClient()
-	if err != nil {
-		return err
-	}
-	errStr := e.ErrorMessage
+func errorsPush(c *cli.Context) error {
+	client := c.App.Metadata["client"].(APIClient)
+	errStr := strings.Join(c.Args(), " ")
 	response, err := client.Post("system/error", strings.TrimSpace(errStr))
 	if err != nil {
 		return err
@@ -48,13 +59,3 @@ func (e *errorsPushCommand) Run(ctx Context) error {
 	}
 	return nil
 }
-
-func (*errorsCommand) Run(ctx Context, kongCtx *kong.Context) error {
-	switch kongCtx.Selected().Name {
-	case "show":
-		return indexDumpOutput("system/error", ctx.clientFactory)
-	case "clear":
-		return emptyPost("system/error/clear", ctx.clientFactory)
-	}
-	return nil
-}

cmd/syncthing/cli/index.go

@@ -7,26 +7,32 @@
 package cli
 
 import (
-	"github.com/alecthomas/kong"
+	"github.com/urfave/cli"
 )
 
-type indexCommand struct {
-	Dump     struct{} `cmd:"" help:"Print the entire db"`
-	DumpSize struct{} `cmd:"" help:"Print the db size of different categories of information"`
-	Check    struct{} `cmd:"" help:"Check the database for inconsistencies"`
-	Account  struct{} `cmd:"" help:"Print key and value size statistics per key type"`
-}
-
-func (*indexCommand) Run(kongCtx *kong.Context) error {
-	switch kongCtx.Selected().Name {
-	case "dump":
-		return indexDump()
-	case "dump-size":
-		return indexDumpSize()
-	case "check":
-		return indexCheck()
-	case "account":
-		return indexAccount()
-	}
-	return nil
+var indexCommand = cli.Command{
+	Name:  "index",
+	Usage: "Show information about the index (database)",
+	Subcommands: []cli.Command{
+		{
+			Name:   "dump",
+			Usage:  "Print the entire db",
+			Action: expects(0, indexDump),
+		},
+		{
+			Name:   "dump-size",
+			Usage:  "Print the db size of different categories of information",
+			Action: expects(0, indexDumpSize),
+		},
+		{
+			Name:   "check",
+			Usage:  "Check the database for inconsistencies",
+			Action: expects(0, indexCheck),
+		},
+		{
+			Name:   "account",
+			Usage:  "Print key and value size statistics per key type",
+			Action: expects(0, indexAccount),
+		},
+	},
 }

cmd/syncthing/cli/index_accounting.go

@@ -10,10 +10,12 @@ import (
 	"fmt"
 	"os"
 	"text/tabwriter"
+
+	"github.com/urfave/cli"
 )
 
 // indexAccount prints key and data size statistics per class
-func indexAccount() error {
+func indexAccount(*cli.Context) error {
 	ldb, err := getDB()
 	if err != nil {
 		return err

cmd/syncthing/cli/index_dump.go

@@ -11,15 +11,13 @@ import (
 	"fmt"
 	"time"
 
-	"google.golang.org/protobuf/proto"
+	"github.com/urfave/cli"
 
-	"github.com/syncthing/syncthing/internal/gen/bep"
-	"github.com/syncthing/syncthing/internal/gen/dbproto"
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func indexDump() error {
+func indexDump(*cli.Context) error {
 	ldb, err := getDB()
 	if err != nil {
 		return err
@@ -37,19 +35,19 @@ func indexDump() error {
 			name := nulString(key[1+4+4:])
 			fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
 
-			var f bep.FileInfo
-			err := proto.Unmarshal(it.Value(), &f)
+			var f protocol.FileInfo
+			err := f.Unmarshal(it.Value())
 			if err != nil {
 				return err
 			}
-			fmt.Printf(" V:%v\n", &f)
+			fmt.Printf(" V:%v\n", f)
 
 		case db.KeyTypeGlobal:
 			folder := binary.BigEndian.Uint32(key[1:])
 			name := nulString(key[1+4:])
-			var flv dbproto.VersionList
-			proto.Unmarshal(it.Value(), &flv)
-			fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, &flv)
+			var flv db.VersionList
+			flv.Unmarshal(it.Value())
+			fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
 
 		case db.KeyTypeBlock:
 			folder := binary.BigEndian.Uint32(key[1:])
@@ -98,11 +96,11 @@ func indexDump() error {
 		case db.KeyTypeFolderMeta:
 			folder := binary.BigEndian.Uint32(key[1:])
 			fmt.Printf("[foldermeta] F:%d", folder)
-			var cs dbproto.CountsSet
-			if err := proto.Unmarshal(it.Value(), &cs); err != nil {
+			var cs db.CountsSet
+			if err := cs.Unmarshal(it.Value()); err != nil {
 				fmt.Printf(" (invalid)\n")
 			} else {
-				fmt.Printf(" V:%v\n", &cs)
+				fmt.Printf(" V:%v\n", cs)
 			}
 
 		case db.KeyTypeMiscData:
@@ -129,20 +127,20 @@ func indexDump() error {
 
 		case db.KeyTypeVersion:
 			fmt.Printf("[version] H:%x", key[1:])
-			var v bep.Vector
-			err := proto.Unmarshal(it.Value(), &v)
+			var v protocol.Vector
+			err := v.Unmarshal(it.Value())
 			if err != nil {
 				fmt.Printf(" (invalid)\n")
 			} else {
-				fmt.Printf(" V:%v\n", &v)
+				fmt.Printf(" V:%v\n", v)
 			}
 
 		case db.KeyTypePendingFolder:
 			device := binary.BigEndian.Uint32(key[1:])
 			folder := string(key[5:])
-			var of dbproto.ObservedFolder
-			proto.Unmarshal(it.Value(), &of)
-			fmt.Printf("[pendingFolder] D:%d F:%s V:%v\n", device, folder, &of)
+			var of db.ObservedFolder
+			of.Unmarshal(it.Value())
+			fmt.Printf("[pendingFolder] D:%d F:%s V:%v\n", device, folder, of)
 
 		case db.KeyTypePendingDevice:
 			device := "<invalid>"
@@ -150,9 +148,9 @@ func indexDump() error {
 			if err == nil {
 				device = dev.String()
 			}
-			var od dbproto.ObservedDevice
-			proto.Unmarshal(it.Value(), &od)
-			fmt.Printf("[pendingDevice] D:%v V:%v\n", device, &od)
+			var od db.ObservedDevice
+			od.Unmarshal(it.Value())
+			fmt.Printf("[pendingDevice] D:%v V:%v\n", device, od)
 
 		default:
 			fmt.Printf("[??? %d]\n  %x\n  %x\n", key[0], key, it.Value())

cmd/syncthing/cli/index_dumpsize.go

@@ -7,33 +7,53 @@
 package cli
 
 import (
+	"container/heap"
 	"encoding/binary"
 	"fmt"
-	"sort"
+
+	"github.com/urfave/cli"
 
 	"github.com/syncthing/syncthing/lib/db"
 )
 
-func indexDumpSize() error {
-	type sizedElement struct {
-		key  string
-		size int
-	}
+type SizedElement struct {
+	key  string
+	size int
+}
 
+type ElementHeap []SizedElement
+
+func (h ElementHeap) Len() int           { return len(h) }
+func (h ElementHeap) Less(i, j int) bool { return h[i].size > h[j].size }
+func (h ElementHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+
+func (h *ElementHeap) Push(x interface{}) {
+	*h = append(*h, x.(SizedElement))
+}
+
+func (h *ElementHeap) Pop() interface{} {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	*h = old[0 : n-1]
+	return x
+}
+
+func indexDumpSize(*cli.Context) error {
 	ldb, err := getDB()
 	if err != nil {
 		return err
 	}
 
+	h := &ElementHeap{}
+	heap.Init(h)
+
 	it, err := ldb.NewPrefixIterator(nil)
 	if err != nil {
 		return err
 	}
-
-	var elems []sizedElement
+	var ele SizedElement
 	for it.Next() {
-		var ele sizedElement
-
 		key := it.Key()
 		switch key[0] {
 		case db.KeyTypeDevice:
@@ -74,13 +94,11 @@ func indexDumpSize() error {
 			ele.key = fmt.Sprintf("UNKNOWN:%x", key)
 		}
 		ele.size = len(it.Value())
-		elems = append(elems, ele)
+		heap.Push(h, ele)
 	}
 
-	sort.Slice(elems, func(i, j int) bool {
-		return elems[i].size > elems[j].size
-	})
-	for _, ele := range elems {
+	for h.Len() > 0 {
+		ele = heap.Pop(h).(SizedElement)
 		fmt.Println(ele.key, ele.size)
 	}
 

cmd/syncthing/cli/index_idxck.go

@@ -13,10 +13,8 @@ import (
 	"fmt"
 	"sort"
 
-	"google.golang.org/protobuf/proto"
+	"github.com/urfave/cli"
 
-	"github.com/syncthing/syncthing/internal/gen/bep"
-	"github.com/syncthing/syncthing/internal/gen/dbproto"
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
@@ -37,7 +35,7 @@ type sequenceKey struct {
 	sequence uint64
 }
 
-func indexCheck() (err error) {
+func indexCheck(*cli.Context) (err error) {
 	ldb, err := getDB()
 	if err != nil {
 		return err
@@ -46,12 +44,12 @@ func indexCheck() (err error) {
 	folders := make(map[uint32]string)
 	devices := make(map[uint32]string)
 	deviceToIDs := make(map[string]uint32)
-	fileInfos := make(map[fileInfoKey]*bep.FileInfo)
-	globals := make(map[globalKey]*dbproto.VersionList)
+	fileInfos := make(map[fileInfoKey]protocol.FileInfo)
+	globals := make(map[globalKey]db.VersionList)
 	sequences := make(map[sequenceKey]string)
 	needs := make(map[globalKey]struct{})
 	blocklists := make(map[string]struct{})
-	versions := make(map[string]*bep.Vector)
+	versions := make(map[string]protocol.Vector)
 	usedBlocklists := make(map[string]struct{})
 	usedVersions := make(map[string]struct{})
 	var localDeviceKey uint32
@@ -59,7 +57,7 @@ func indexCheck() (err error) {
 	defer func() {
 		if err == nil {
 			if success {
-				fmt.Println("Index check completed successfully.")
+				fmt.Println("Index check completed succesfully.")
 			} else {
 				err = errors.New("Inconsistencies found in the index")
 			}
@@ -78,26 +76,26 @@ func indexCheck() (err error) {
 			device := binary.BigEndian.Uint32(key[1+4:])
 			name := nulString(key[1+4+4:])
 
-			var f bep.FileInfo
-			err := proto.Unmarshal(it.Value(), &f)
+			var f protocol.FileInfo
+			err := f.Unmarshal(it.Value())
 			if err != nil {
 				fmt.Println("Unable to unmarshal FileInfo:", err)
 				success = false
 				continue
 			}
 
-			fileInfos[fileInfoKey{folder, device, name}] = &f
+			fileInfos[fileInfoKey{folder, device, name}] = f
 
 		case db.KeyTypeGlobal:
 			folder := binary.BigEndian.Uint32(key[1:])
 			name := nulString(key[1+4:])
-			var flv dbproto.VersionList
-			if err := proto.Unmarshal(it.Value(), &flv); err != nil {
+			var flv db.VersionList
+			if err := flv.Unmarshal(it.Value()); err != nil {
 				fmt.Println("Unable to unmarshal VersionList:", err)
 				success = false
 				continue
 			}
-			globals[globalKey{folder, name}] = &flv
+			globals[globalKey{folder, name}] = flv
 
 		case db.KeyTypeFolderIdx:
 			key := binary.BigEndian.Uint32(it.Key()[1:])
@@ -128,13 +126,13 @@ func indexCheck() (err error) {
 
 		case db.KeyTypeVersion:
 			hash := string(key[1:])
-			var v bep.Vector
-			if err := proto.Unmarshal(it.Value(), &v); err != nil {
+			var v protocol.Vector
+			if err := v.Unmarshal(it.Value()); err != nil {
 				fmt.Println("Unable to unmarshal Vector:", err)
 				success = false
 				continue
 			}
-			versions[hash] = &v
+			versions[hash] = v
 		}
 	}
 
@@ -252,27 +250,25 @@ func indexCheck() (err error) {
 			if fi.VersionHash != nil {
 				fiv = versions[string(fi.VersionHash)]
 			}
-			if !protocol.VectorFromWire(fiv).Equal(version) {
+			if !fiv.Equal(version) {
 				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, version, fi.Version)
 				success = false
 			}
-			ffi := protocol.FileInfoFromDB(fi)
-			if ffi.IsInvalid() != invalid {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, invalid, ffi.IsInvalid())
+			if fi.IsInvalid() != invalid {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, invalid, fi.IsInvalid())
 				success = false
 			}
-			if ffi.IsDeleted() != deleted {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo deleted mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, deleted, ffi.IsDeleted())
+			if fi.IsDeleted() != deleted {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo deleted mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, deleted, fi.IsDeleted())
 				success = false
 			}
 		}
-		for i, fv := range vl.Versions {
-			ver := protocol.VectorFromWire(fv.Version)
+		for i, fv := range vl.RawVersions {
 			for _, device := range fv.Devices {
-				checkGlobal(i, device, ver, false, fv.Deleted)
+				checkGlobal(i, device, fv.Version, false, fv.Deleted)
 			}
 			for _, device := range fv.InvalidDevices {
-				checkGlobal(i, device, ver, true, fv.Deleted)
+				checkGlobal(i, device, fv.Version, true, fv.Deleted)
 			}
 		}
 
@@ -282,10 +278,10 @@ func indexCheck() (err error) {
 		if needsLocally(vl) {
 			_, ok := needs[gk]
 			if !ok {
-				fv, _ := vlGetGlobal(vl)
-				devB, _ := fvFirstDevice(fv)
+				fv, _ := vl.GetGlobal()
+				devB, _ := fv.FirstDevice()
 				dev := deviceToIDs[string(devB)]
-				fi := protocol.FileInfoFromDB(fileInfos[fileInfoKey{gk.folder, dev, gk.name}])
+				fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
 				if !fi.IsDeleted() && !fi.IsIgnored() {
 					fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
 				}
@@ -351,84 +347,11 @@ func indexCheck() (err error) {
 	return nil
 }
 
-func needsLocally(vl *dbproto.VersionList) bool {
-	gfv, gok := vlGetGlobal(vl)
-	if !gok { // That's weird, but we hardly need something non-existent
+func needsLocally(vl db.VersionList) bool {
+	gfv, gok := vl.GetGlobal()
+	if !gok { // That's weird, but we hardly need something non-existant
 		return false
 	}
-	fv, ok := vlGet(vl, protocol.LocalDeviceID[:])
-	return db.Need(gfv, ok, protocol.VectorFromWire(fv.Version))
-}
-
-// Get returns a FileVersion that contains the given device and whether it has
-// been found at all.
-func vlGet(vl *dbproto.VersionList, device []byte) (*dbproto.FileVersion, bool) {
-	_, i, _, ok := vlFindDevice(vl, device)
-	if !ok {
-		return &dbproto.FileVersion{}, false
-	}
-	return vl.Versions[i], true
-}
-
-// GetGlobal returns the current global FileVersion. The returned FileVersion
-// may be invalid, if all FileVersions are invalid. Returns false only if
-// VersionList is empty.
-func vlGetGlobal(vl *dbproto.VersionList) (*dbproto.FileVersion, bool) {
-	i := vlFindGlobal(vl)
-	if i == -1 {
-		return nil, false
-	}
-	return vl.Versions[i], true
-}
-
-// findGlobal returns the first version that isn't invalid, or if all versions are
-// invalid just the first version (i.e. 0) or -1, if there's no versions at all.
-func vlFindGlobal(vl *dbproto.VersionList) int {
-	for i := range vl.Versions {
-		if !fvIsInvalid(vl.Versions[i]) {
-			return i
-		}
-	}
-	if len(vl.Versions) == 0 {
-		return -1
-	}
-	return 0
-}
-
-// findDevice returns whether the device is in InvalidVersions or Versions and
-// in InvalidDevices or Devices (true for invalid), the positions in the version
-// and device slices and whether it has been found at all.
-func vlFindDevice(vl *dbproto.VersionList, device []byte) (bool, int, int, bool) {
-	for i, v := range vl.Versions {
-		if j := deviceIndex(v.Devices, device); j != -1 {
-			return false, i, j, true
-		}
-		if j := deviceIndex(v.InvalidDevices, device); j != -1 {
-			return true, i, j, true
-		}
-	}
-	return false, -1, -1, false
-}
-
-func deviceIndex(devices [][]byte, device []byte) int {
-	for i, dev := range devices {
-		if bytes.Equal(device, dev) {
-			return i
-		}
-	}
-	return -1
-}
-
-func fvFirstDevice(fv *dbproto.FileVersion) ([]byte, bool) {
-	if len(fv.Devices) != 0 {
-		return fv.Devices[0], true
-	}
-	if len(fv.InvalidDevices) != 0 {
-		return fv.InvalidDevices[0], true
-	}
-	return nil, false
-}
-
-func fvIsInvalid(fv *dbproto.FileVersion) bool {
-	return fv == nil || len(fv.Devices) == 0
+	fv, ok := vl.Get(protocol.LocalDeviceID[:])
+	return db.Need(gfv, ok, fv.Version)
 }

cmd/syncthing/cli/main.go

@@ -9,87 +9,152 @@ package cli
 import (
 	"bufio"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"strings"
 
 	"github.com/alecthomas/kong"
-	"github.com/kballard/go-shellquote"
+	"github.com/flynn-archive/go-shlex"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
 
 	"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
 	"github.com/syncthing/syncthing/lib/config"
 )
 
-type CLI struct {
-	cmdutil.CommonOptions
-	DataDir    string `name:"data" placeholder:"PATH" env:"STDATADIR" help:"Set data directory (database and logs)"`
+type preCli struct {
 	GUIAddress string `name:"gui-address"`
 	GUIAPIKey  string `name:"gui-apikey"`
-
-	Show       showCommand      `cmd:"" help:"Show command group"`
-	Debug      debugCommand     `cmd:"" help:"Debug command group"`
-	Operations operationCommand `cmd:"" help:"Operation command group"`
-	Errors     errorsCommand    `cmd:"" help:"Error command group"`
-	Config     configCommand    `cmd:"" help:"Configuration modification command group" passthrough:""`
-	Stdin      stdinCommand     `cmd:"" name:"-" help:"Read commands from stdin"`
+	HomeDir    string `name:"home"`
+	ConfDir    string `name:"config"`
+	DataDir    string `name:"data"`
 }
 
-type Context struct {
-	clientFactory *apiClientFactory
-}
+func Run() error {
+	// This is somewhat a hack around a chicken and egg problem. We need to set
+	// the home directory and potentially other flags to know where the
+	// syncthing instance is running in order to get it's config ... which we
+	// then use to construct the actual CLI ... at which point it's too late to
+	// add flags there...
+	c := preCli{}
+	parseFlags(&c)
 
-func (cli CLI) AfterApply(kongCtx *kong.Context) error {
-	err := cmdutil.SetConfigDataLocationsFromFlags(cli.HomeDir, cli.ConfDir, cli.DataDir)
+	// Not set as default above because the strings can be really long.
+	err := cmdutil.SetConfigDataLocationsFromFlags(c.HomeDir, c.ConfDir, c.DataDir)
 	if err != nil {
-		return fmt.Errorf("command line options: %w", err)
+		return errors.Wrap(err, "Command line options:")
 	}
-
 	clientFactory := &apiClientFactory{
 		cfg: config.GUIConfiguration{
-			RawAddress: cli.GUIAddress,
-			APIKey:     cli.GUIAPIKey,
+			RawAddress: c.GUIAddress,
+			APIKey:     c.GUIAPIKey,
 		},
 	}
 
-	context := Context{
-		clientFactory: clientFactory,
+	configCommand, err := getConfigCommand(clientFactory)
+	if err != nil {
+		return err
 	}
 
-	kongCtx.Bind(context)
-	return nil
+	// Implement the same flags at the upper CLI, but do nothing with them.
+	// This is so that the usage text is the same
+	fakeFlags := []cli.Flag{
+		cli.StringFlag{
+			Name:  "gui-address",
+			Value: "URL",
+			Usage: "Override GUI address (e.g. \"http://192.0.2.42:8443\")",
+		},
+		cli.StringFlag{
+			Name:  "gui-apikey",
+			Value: "API-KEY",
+			Usage: "Override GUI API key",
+		},
+		cli.StringFlag{
+			Name:  "home",
+			Value: "PATH",
+			Usage: "Set configuration and data directory",
+		},
+		cli.StringFlag{
+			Name:  "conf",
+			Value: "PATH",
+			Usage: "Set configuration directory (config and keys)",
+		},
+	}
+
+	// Construct the actual CLI
+	app := cli.NewApp()
+	app.Author = "The Syncthing Authors"
+	app.Metadata = map[string]interface{}{
+		"clientFactory": clientFactory,
+	}
+	app.Commands = []cli.Command{{
+		Name:  "cli",
+		Usage: "Syncthing command line interface",
+		Flags: fakeFlags,
+		Subcommands: []cli.Command{
+			configCommand,
+			showCommand,
+			operationCommand,
+			errorsCommand,
+			debugCommand,
+			{
+				Name:     "-",
+				HideHelp: true,
+				Usage:    "Read commands from stdin",
+				Action: func(ctx *cli.Context) error {
+					if ctx.NArg() > 0 {
+						return errors.New("command does not expect any arguments")
+					}
+
+					// Drop the `-` not to recurse into self.
+					args := make([]string, len(os.Args)-1)
+					copy(args, os.Args)
+
+					fmt.Println("Reading commands from stdin...", args)
+					scanner := bufio.NewScanner(os.Stdin)
+					for scanner.Scan() {
+						input, err := shlex.Split(scanner.Text())
+						if err != nil {
+							return errors.Wrap(err, "parsing input")
+						}
+						if len(input) == 0 {
+							continue
+						}
+						err = app.Run(append(args, input...))
+						if err != nil {
+							return err
+						}
+					}
+					return scanner.Err()
+				},
+			},
+		},
+	}}
+
+	return app.Run(os.Args)
 }
 
-type stdinCommand struct{}
-
-func (*stdinCommand) Run() error {
-	// Drop the `-` not to recurse into self.
-	args := make([]string, len(os.Args)-1)
-	copy(args, os.Args)
-
-	fmt.Println("Reading commands from stdin...", args)
-	scanner := bufio.NewScanner(os.Stdin)
-	for scanner.Scan() {
-		input, err := shellquote.Split(scanner.Text())
-		if err != nil {
-			return fmt.Errorf("parsing input: %w", err)
+func parseFlags(c *preCli) error {
+	// kong only needs to parse the global arguments after "cli" and before the
+	// subcommand (if any).
+	if len(os.Args) <= 2 {
+		return nil
+	}
+	args := os.Args[2:]
+	for i := 0; i < len(args); i++ {
+		if !strings.HasPrefix(args[i], "--") {
+			args = args[:i]
+			break
 		}
-		if len(input) == 0 {
-			continue
-		}
-
-		var cli CLI
-		p, err := kong.New(&cli)
-		if err != nil {
-			// can't happen, really
-			return fmt.Errorf("creating parser: %w", err)
-		}
-		ctx, err := p.Parse(input)
-		if err != nil {
-			fmt.Println("Error:", err)
-			continue
-		}
-		if err := ctx.Run(); err != nil {
-			fmt.Println("Error:", err)
-			continue
+		if !strings.Contains(args[i], "=") {
+			i++
 		}
 	}
-	return scanner.Err()
+	// We don't want kong to print anything nor os.Exit (e.g. on -h)
+	parser, err := kong.New(c, kong.Writers(ioutil.Discard, ioutil.Discard), kong.Exit(func(int) {}))
+	if err != nil {
+		return err
+	}
+	_, err = parser.Parse(args)
+	return err
 }

cmd/syncthing/cli/operations.go

@@ -7,48 +7,42 @@
 package cli
 
 import (
-	"bufio"
-	"errors"
 	"fmt"
-	"path/filepath"
 
-	"github.com/alecthomas/kong"
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/fs"
+	"github.com/urfave/cli"
 )
 
-type folderOverrideCommand struct {
-	FolderID string `arg:""`
+var operationCommand = cli.Command{
+	Name:     "operations",
+	HideHelp: true,
+	Usage:    "Operation command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "restart",
+			Usage:  "Restart syncthing",
+			Action: expects(0, emptyPost("system/restart")),
+		},
+		{
+			Name:   "shutdown",
+			Usage:  "Shutdown syncthing",
+			Action: expects(0, emptyPost("system/shutdown")),
+		},
+		{
+			Name:   "upgrade",
+			Usage:  "Upgrade syncthing (if a newer version is available)",
+			Action: expects(0, emptyPost("system/upgrade")),
+		},
+		{
+			Name:      "folder-override",
+			Usage:     "Override changes on folder (remote for sendonly, local for receiveonly). WARNING: Destructive - deletes/changes your data.",
+			ArgsUsage: "[folder id]",
+			Action:    expects(1, foldersOverride),
+		},
+	},
 }
 
-type defaultIgnoresCommand struct {
-	Path string `arg:""`
-}
-
-type operationCommand struct {
-	Restart        struct{}              `cmd:"" help:"Restart syncthing"`
-	Shutdown       struct{}              `cmd:"" help:"Shutdown syncthing"`
-	Upgrade        struct{}              `cmd:"" help:"Upgrade syncthing (if a newer version is available)"`
-	FolderOverride folderOverrideCommand `cmd:"" help:"Override changes on folder (remote for sendonly, local for receiveonly). WARNING: Destructive - deletes/changes your data"`
-	DefaultIgnores defaultIgnoresCommand `cmd:"" help:"Set the default ignores (config) from a file"`
-}
-
-func (*operationCommand) Run(ctx Context, kongCtx *kong.Context) error {
-	f := ctx.clientFactory
-
-	switch kongCtx.Selected().Name {
-	case "restart":
-		return emptyPost("system/restart", f)
-	case "shutdown":
-		return emptyPost("system/shutdown", f)
-	case "upgrade":
-		return emptyPost("system/upgrade", f)
-	}
-	return nil
-}
-
-func (f *folderOverrideCommand) Run(ctx Context) error {
-	client, err := ctx.clientFactory.getClient()
+func foldersOverride(c *cli.Context) error {
+	client, err := getClientFactory(c).getClient()
 	if err != nil {
 		return err
 	}
@@ -56,7 +50,7 @@ func (f *folderOverrideCommand) Run(ctx Context) error {
 	if err != nil {
 		return err
 	}
-	rid := f.FolderID
+	rid := c.Args()[0]
 	for _, folder := range cfg.Folders {
 		if folder.ID == rid {
 			response, err := client.Post("db/override", "")
@@ -73,36 +67,10 @@ func (f *folderOverrideCommand) Run(ctx Context) error {
 				if body != "" {
 					errStr += "\nBody: " + body
 				}
-				return errors.New(errStr)
+				return fmt.Errorf(errStr)
 			}
 			return nil
 		}
 	}
-	return fmt.Errorf("Folder %q not found", rid)
-}
-
-func (d *defaultIgnoresCommand) Run(ctx Context) error {
-	client, err := ctx.clientFactory.getClient()
-	if err != nil {
-		return err
-	}
-	dir, file := filepath.Split(d.Path)
-	filesystem := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
-
-	fd, err := filesystem.Open(file)
-	if err != nil {
-		return err
-	}
-	scanner := bufio.NewScanner(fd)
-	var lines []string
-	for scanner.Scan() {
-		lines = append(lines, scanner.Text())
-	}
-	fd.Close()
-	if err := scanner.Err(); err != nil {
-		return err
-	}
-
-	_, err = client.PutJSON("config/defaults/ignores", config.Ignores{Lines: lines})
-	return err
+	return fmt.Errorf("Folder " + rid + " not found")
 }

cmd/syncthing/cli/pending.go

@@ -1,38 +0,0 @@
-// Copyright (C) 2021 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package cli
-
-import (
-	"net/url"
-
-	"github.com/alecthomas/kong"
-)
-
-type pendingCommand struct {
-	Devices struct{} `cmd:"" help:"Show pending devices"`
-	Folders struct {
-		Device string `help:"Show pending folders offered by given device"`
-	} `cmd:"" help:"Show pending folders"`
-}
-
-func (p *pendingCommand) Run(ctx Context, kongCtx *kong.Context) error {
-	indexDumpOutput := indexDumpOutputWrapper(ctx.clientFactory)
-
-	switch kongCtx.Selected().Name {
-	case "devices":
-		return indexDumpOutput("cluster/pending/devices")
-	case "folders":
-		if p.Folders.Device != "" {
-			query := make(url.Values)
-			query.Set("device", p.Folders.Device)
-			return indexDumpOutput("cluster/pending/folders?" + query.Encode())
-		}
-		return indexDumpOutput("cluster/pending/folders")
-	}
-
-	return nil
-}

cmd/syncthing/cli/show.go

@@ -7,36 +7,38 @@
 package cli
 
 import (
-	"github.com/alecthomas/kong"
+	"github.com/urfave/cli"
 )
 
-type showCommand struct {
-	Version      struct{}       `cmd:"" help:"Show syncthing client version"`
-	ConfigStatus struct{}       `cmd:"" help:"Show configuration status, whether or not a restart is required for changes to take effect"`
-	System       struct{}       `cmd:"" help:"Show system status"`
-	Connections  struct{}       `cmd:"" help:"Report about connections to other devices"`
-	Discovery    struct{}       `cmd:"" help:"Show the discovered addresses of remote devices (from cache of the running syncthing instance)"`
-	Usage        struct{}       `cmd:"" help:"Show usage report"`
-	Pending      pendingCommand `cmd:"" help:"Pending subcommand group"`
-}
-
-func (*showCommand) Run(ctx Context, kongCtx *kong.Context) error {
-	indexDumpOutput := indexDumpOutputWrapper(ctx.clientFactory)
-
-	switch kongCtx.Selected().Name {
-	case "version":
-		return indexDumpOutput("system/version")
-	case "config-status":
-		return indexDumpOutput("config/restart-required")
-	case "system":
-		return indexDumpOutput("system/status")
-	case "connections":
-		return indexDumpOutput("system/connections")
-	case "discovery":
-		return indexDumpOutput("system/discovery")
-	case "usage":
-		return indexDumpOutput("svc/report")
-	}
-
-	return nil
+var showCommand = cli.Command{
+	Name:     "show",
+	HideHelp: true,
+	Usage:    "Show command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "version",
+			Usage:  "Show syncthing client version",
+			Action: expects(0, indexDumpOutput("system/version")),
+		},
+		{
+			Name:   "config-status",
+			Usage:  "Show configuration status, whether or not a restart is required for changes to take effect",
+			Action: expects(0, indexDumpOutput("config/restart-required")),
+		},
+		{
+			Name:   "system",
+			Usage:  "Show system status",
+			Action: expects(0, indexDumpOutput("system/status")),
+		},
+		{
+			Name:   "connections",
+			Usage:  "Report about connections to other devices",
+			Action: expects(0, indexDumpOutput("system/connections")),
+		},
+		{
+			Name:   "usage",
+			Usage:  "Show usage report",
+			Action: expects(0, indexDumpOutput("svc/report")),
+		},
+	},
 }