lib/db: Add hacky way to adjust database parameters (#5889)

This adds a set of magical environment variables that can be used to
tweak the database parameters. It's totally undocumented and not
intended to be a long term or supported thing.

It's ugly, but there is a backstory. I have a couple of large
installations where the database options are inefficient or otherwise
suboptimal (24/7 compaction going on and stuff like that). I don't
*know* the correct database parameters, nor yet the formula or method to
derive them by, so this requires experimentation. Experimentation needs
to happen partly in production, and rolling out new builds for every
tweak isn't practical. This provides override points for all reasonable
values, while not changing anything by default.

Ideally, at the end of such experimentation, we'll know which values are
relevant to change and in what manner, and can provide a more user
friendly knob to do so - or do it automatically based on the database
size.

.codecov.yml

@@ -1,20 +0,0 @@
-comment: false
-
-coverage:
-  range: "40...100"
-  precision: 1
-  status:
-    patch:
-      default:
-        informational: true
-    project:
-      default:
-        informational: true
-
-github_checks:
-  annotations: false
-
-ignore:
-  - "**.pb.go"
-  - "**_mocked.go"
-  - "**/mocks/*"

.deepsource.toml

@@ -1,12 +0,0 @@
-version = 1
-
-exclude_patterns = ["**/*.pb.go"]
-test_patterns = ["**/*_test.go"]
-
-[[analyzers]]
-name = "go"
-enabled = true
-
-  [analyzers.meta]
-  import_paths = ["github.com/syncthing/syncthing"]
-  build_tags = ["noassets"]

.github/FUNDING.yml

@@ -1,11 +0,0 @@
-github: syncthing
-custom: "https://syncthing.net/donations/"
-
-# patreon: # Replace with a single Patreon username
-# open_collective: # Replace with a single Open Collective username
-# ko_fi: # Replace with a single Ko-fi username
-# tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-# community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-# liberapay: # Replace with a single Liberapay username
-# issuehunt: # Replace with a single IssueHunt username
-# otechie: # Replace with a single Otechie username

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,42 @@
+### DO NOT REPORT SECURITY ISSUES IN THIS ISSUE TRACKER
+
+Instead, contact security@syncthing.net directly - see
+https://syncthing.net/security.html for more information.
+
+### DO NOT POST SUPPORT REQUESTS OR GENERAL QUESTIONS IN THIS ISSUE TRACKER
+
+Please use the forum at https://forum.syncthing.net/ where a large number of
+helpful people hang out. This issue tracker is for reporting bugs or feature
+requests directly to the developers. Worst case you might get a short
+"that's a bug, please report it on GitHub" response on the forum, in which
+case we thank you for your patience and following our advice. :)
+
+### Please use the correct issue tracker
+
+If your problem relates to a Syncthing wrapper or [sub-project](https://github.com/syncthing) such as [Syncthing for Android](https://github.com/syncthing/syncthing-android/issues), [SyncTrayzor](https://github.com/canton7/synctrayzor) or the [documentation](https://github.com/syncthing/docs/issues), please use their respective issue trackers.
+
+### Does your log mention database corruption?
+
+If your Syncthing log reports panics because of database corruption it is most likely a fault with your system's storage or memory. Affected log entries will contain lines starting with `panic: leveldb`. You will need to delete the index database to clear this, by running `syncthing -reset-database`.
+
+### Please do post actual bug reports and feature requests.
+
+If your issue is a bug report, replace this boilerplate with a description
+of the problem, being sure to include at least:
+
+ - what happened,
+ - what you expected to happen instead, and
+ - any steps to reproduce the problem.
+
+Also fill out the version information below and add log output or
+screenshots as appropriate.
+
+If your issue is a feature request, simply replace this template text in
+its entirety.
+
+### Version Information
+
+Syncthing Version: v0.x.y
+OS Version: Windows 7 / Ubuntu 14.04 / ...
+Browser Version: (if applicable, for GUI issues)
+

.github/ISSUE_TEMPLATE/01-feature.md

@@ -1,13 +0,0 @@
----
-name: Feature request
-about: If you're just not sure how to do something, see "ask a question".
-labels: enhancement, needs-triage
----
-
-### Include required information
-
-Please be sure to include at least:
-
- - what problem your new feature would solve
- - how or why you think it is generally useful (i.e., not just for you)
- - what alternatives or workarounds you considered

.github/ISSUE_TEMPLATE/02-bug.md

@@ -1,23 +0,0 @@
----
-name: Bug report
-about: If you're actually looking for support, see "ask a question".
-labels: bug, needs-triage
----
-
-### Does your log mention database corruption?
-
-If your Syncthing log reports panics because of database corruption it is
-most likely a fault with your system's storage or memory. Affected log
-entries will contain lines starting with `panic: leveldb`. You will need to
-delete the index database to clear this, by running `syncthing
--reset-database`.
-
-### Include required information
-
-Please be sure to include at least:
-
- - which version of Syncthing and what operating system you are using
- - browser and version, if applicable
- - what happened,
- - what you expected to happen instead, and
- - any steps to reproduce the problem.

.github/ISSUE_TEMPLATE/config.yml

@@ -1,8 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: I need help / I have a question
-    url: https://forum.syncthing.net/
-    about: Ask questions, get support, and discuss with other community members.
-  - name: Android issues
-    url: https://github.com/syncthing/syncthing-android/issues/
-    about: The Android app has its own issue tracker.

.github/SECURITY.md

@@ -1,10 +0,0 @@
-## Reporting a Vulnerability
-
-If you believe that you've found a Syncthing-related security vulnerability,
-please report it by sending email to the address security@syncthing.net. The
-[PGP key for security@syncthing.net
-(B683AD7B76CAB013)](https://syncthing.net/security-key.txt) can be used to
-send encrypted mail or to verify responses received from that address.
-
-You can read more about Syncthing security at
-https://syncthing.net/security/.

.github/dependabot.yml

@@ -1,13 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: weekly
-    open-pull-requests-limit: 10
-
-  - package-ecosystem: "gomod"
-    directory: "/"
-    schedule:
-      interval: weekly
-    open-pull-requests-limit: 10

.github/workflows/build-infra-dockers.yaml

@@ -1,54 +0,0 @@
-name: Build Infrastructure Images
-
-on:
-  push:
-    branches:
-      - infrastructure
-
-env:
-    GO_VERSION: "~1.21.1"
-    CGO_ENABLED: "0"
-    BUILD_USER: docker
-    BUILD_HOST: github.syncthing.net
-
-jobs:
-
-  docker-syncthing:
-    name: Build and push Docker images
-    runs-on: ubuntu-latest
-    environment: docker
-    strategy:
-        matrix:
-            pkg:
-                - stcrashreceiver
-                - strelaypoolsrv
-                - stupgrades
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Build binaries
-        run: |
-          for arch in arm64 amd64; do
-            go run build.go -goos linux -goarch "$arch" build ${{ matrix.pkg }}
-            mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-"$arch"
-          done
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./Dockerfile.${{ matrix.pkg }}
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: syncthing/${{ matrix.pkg }}:latest,syncthing/${{ matrix.pkg }}:${{ github.sha }}

.github/workflows/build-syncthing.yaml

@@ -1,811 +0,0 @@
-name: Build Syncthing
-
-on:
-  pull_request:
-  push:
-  schedule:
-    # Run nightly build at 05:00 UTC
-    - cron: '00 05 * * *'
-  workflow_dispatch:
-
-env:
-  # The go version to use for builds. We set check-latest to true when
-  # installing, so we get the latest patch version that matches the
-  # expression.
-  GO_VERSION: "~1.21.1"
-
-  # Optimize compatibility on the slow archictures.
-  GO386: softfloat
-  GOARM: "5"
-  GOMIPS: softfloat
-
-  # Avoid hilarious amounts of obscuring log output when running tests.
-  LOGGER_DISCARD: "1"
-
-  # Our build metadata
-  BUILD_USER: builder
-  BUILD_HOST: github.syncthing.net
-
-# A note on actions and third party code... The actions under actions/ (like
-# `uses: actions/checkout`) are maintained by GitHub, and we need to trust
-# GitHub to maintain their code and infrastructure or we're in deep shit in
-# general. The same doesn't necessarily apply to other actions authors, so
-# some care needs to be taken when adding steps, especially in the paths
-# that lead up to code being packaged and signed.
-
-jobs:
-
-  #
-  # Tests for all platforms. Runs a matrix build on Windows, Linux and Mac,
-  # with the list of expected supported Go versions (current, previous).
-  #
-
-  build-test:
-    name: Build and test
-    strategy:
-      fail-fast: false
-      matrix:
-        runner: ["windows-latest", "ubuntu-latest", "macos-latest"]
-        # The oldest version in this list should match what we have in our go.mod.
-        # Variables don't seem to be supported here, or we could have done something nice.
-        go: ["1.20", "1.21"]
-
-        # Don't run the Windows tests with Go 1.21.4 or 1.20.11; this can be
-        # removed when 1.21.5 and 1.20.12 is released.
-        exclude:
-          - runner: windows-latest
-            go: "1.20"
-          - runner: windows-latest
-            go: "1.21"
-        include:
-          - runner: windows-latest
-            go: "~1.20.12 || ~1.20.0 <1.20.11"
-          - runner: windows-latest
-            go: "~1.21.5 || ~1.21.1 <1.21.4"
-    runs-on: ${{ matrix.runner }}
-    steps:
-      - name: Set git to use LF
-        if: matrix.runner == 'windows-latest'
-        # Without this, the Windows checkout will happen with CRLF line
-        # endings, which is fine for the source code but messes up tests
-        # that depend on data on disk being as expected. Ideally, those
-        # tests should be fixed, but not today.
-        run: |
-          git config --global core.autocrlf false
-          git config --global core.eol lf
-
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ matrix.go }}
-          cache: true
-          check-latest: true
-
-      - name: Build
-        run: |
-          go run build.go
-
-      - name: Install go-test-json-to-loki
-        run: |
-          go install calmh.dev/go-test-json-to-loki@latest
-
-      - name: Test
-        run: |
-          go version
-          go run build.go test | go-test-json-to-loki
-        env:
-          GOFLAGS: "-json"
-          LOKI_URL: ${{ vars.LOKI_URL }}
-          LOKI_USER: ${{ vars.LOKI_USER }}
-          LOKI_PASSWORD: ${{ secrets.LOKI_PASSWORD }}
-          LOKI_LABELS: "go=${{ matrix.go }},runner=${{ matrix.runner }},repo=${{ github.repository }},ref=${{ github.ref }}"
-
-  #
-  # Meta checks for formatting, copyright, etc
-  #
-
-  correctness:
-    name: Check correctness
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Check correctness
-        run: |
-          go test -v ./meta
-
-  #
-  # The basic checks job is a virtual one that depends on the matrix tests,
-  # the correctness checks, and various builds that we always do. This makes
-  # it easy to have the PR process have a single test as a gatekeeper for
-  # merging, instead of having to add all the matrix tests and update them
-  # each time the version changes. (The top level test is not available for
-  # choosing there, only the matrix "children".)
-  #
-
-  basics:
-    name: Basic checks passed
-    runs-on: ubuntu-latest
-    needs:
-      - build-test
-      - correctness
-      - package-linux
-      - package-cross
-      - package-source
-      - package-debian
-      - govulncheck
-    steps:
-      - uses: actions/checkout@v4
-
-  #
-  # Windows
-  #
-
-  package-windows:
-    name: Package for Windows
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
-    environment: signing
-    runs-on: windows-latest
-    steps:
-      - name: Set git to use LF
-        # Without this, the checkout will happen with CRLF line endings,
-        # which is fine for the source code but messes up tests that depend
-        # on data on disk being as expected. Ideally, those tests should be
-        # fixed, but not today.
-        run: |
-          git config --global core.autocrlf false
-          git config --global core.eol lf
-
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v4
-        with:
-          # Temporary version constraint to avoid 1.21.4 which has a bug in
-          # path handling. This can be removed when 1.21.5 is released.
-          go-version: "~1.21.5 || ~1.21.1 <1.21.4"
-          cache: false
-          check-latest: true
-
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~\AppData\Local\go-build
-            ~\go\pkg\mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
-
-      - name: Install dependencies
-        run: |
-          go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@v1.4.0
-
-      - name: Create packages
-        run: |
-          go run build.go -goarch amd64 zip
-          go run build.go -goarch arm zip
-          go run build.go -goarch arm64 zip
-          go run build.go -goarch 386 zip
-        env:
-          CGO_ENABLED: "0"
-          CODESIGN_SIGNTOOL: ${{ secrets.CODESIGN_SIGNTOOL }}
-          CODESIGN_CERTIFICATE_BASE64: ${{ secrets.CODESIGN_CERTIFICATE_BASE64 }}
-          CODESIGN_CERTIFICATE_PASSWORD: ${{ secrets.CODESIGN_CERTIFICATE_PASSWORD }}
-          CODESIGN_TIMESTAMP_SERVER: ${{ secrets.CODESIGN_TIMESTAMP_SERVER }}
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: packages-windows
-          path: syncthing-windows-*.zip
-
-  #
-  # Linux
-  #
-
-  package-linux:
-    name: Package for Linux
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
-
-      - name: Create packages
-        run: |
-          archs=$(go tool dist list | grep linux | sed 's#linux/##')
-          for goarch in $archs ; do
-            go run build.go -goarch "$goarch" tar
-          done
-        env:
-          CGO_ENABLED: "0"
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: packages-linux
-          path: syncthing-linux-*.tar.gz
-
-  #
-  # macOS
-  #
-
-  package-macos:
-    name: Package for macOS
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
-    environment: signing
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
-
-      - name: Import signing certificate
-        run: |
-          # Set up a run-specific keychain, making it available for the
-          # `codesign` tool.
-          umask 066
-          KEYCHAIN_PATH=$RUNNER_TEMP/codesign.keychain
-          KEYCHAIN_PASSWORD=$(uuidgen)
-          security create-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
-          security default-keychain -s "$KEYCHAIN_PATH"
-          security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
-          security set-keychain-settings -lut 21600 "$KEYCHAIN_PATH"
-
-          # Import the certificate
-          CERTIFICATE_PATH=$RUNNER_TEMP/codesign.p12
-          echo "$DEVELOPER_ID_CERTIFICATE_BASE64" | base64 -d -o "$CERTIFICATE_PATH"
-          security import "$CERTIFICATE_PATH" -k "$KEYCHAIN_PATH" -P "$DEVELOPER_ID_CERTIFICATE_PASSWORD" -T /usr/bin/codesign -T /usr/bin/productsign
-          security set-key-partition-list -S apple-tool:,apple: -s -k actions "$KEYCHAIN_PATH"
-
-          # Set the codesign identity for following steps
-          echo "CODESIGN_IDENTITY=$CODESIGN_IDENTITY" >> $GITHUB_ENV
-        env:
-          DEVELOPER_ID_CERTIFICATE_BASE64: ${{ secrets.DEVELOPER_ID_CERTIFICATE_BASE64 }}
-          DEVELOPER_ID_CERTIFICATE_PASSWORD: ${{ secrets.DEVELOPER_ID_CERTIFICATE_PASSWORD }}
-          CODESIGN_IDENTITY: ${{ secrets.CODESIGN_IDENTITY }}
-
-      - name: Create package (amd64)
-        run: |
-          go run build.go -goarch amd64 zip
-        env:
-          CGO_ENABLED: "1"
-
-      - name: Create package (arm64 cross)
-        run: |
-          cat <<EOT > xgo.sh
-          #!/bin/bash
-          CGO_ENABLED=1 \
-            CGO_CFLAGS="-target arm64-apple-macos10.15" \
-            CGO_LDFLAGS="-target arm64-apple-macos10.15" \
-            go "\$@"
-          EOT
-          chmod 755 xgo.sh
-          go run build.go -gocmd ./xgo.sh -goarch arm64 zip
-        env:
-          CGO_ENABLED: "1"
-
-      - name: Create package (universal)
-        run: |
-          rm -rf _tmp
-          mkdir _tmp
-          pushd _tmp
-
-          unzip ../syncthing-macos-amd64-*.zip
-          unzip ../syncthing-macos-arm64-*.zip
-          lipo -create syncthing-macos-amd64-*/syncthing syncthing-macos-arm64-*/syncthing -o syncthing
-
-          amd64=(syncthing-macos-amd64-*)
-          universal="${amd64/amd64/universal}"
-          mv "$amd64" "$universal"
-          mv syncthing "$universal"
-          zip -r "../$universal.zip" "$universal"
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: packages-macos
-          path: syncthing-*.zip
-
-  notarize-macos:
-    name: Notarize for macOS
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
-    environment: signing
-    needs:
-      - package-macos
-      - basics
-    runs-on: macos-latest
-    steps:
-      - name: Download artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: packages-macos
-
-      - name: Notarize binaries
-        run: |
-          APPSTORECONNECT_API_KEY_PATH="$RUNNER_TEMP/apikey.p8"
-          echo "$APPSTORECONNECT_API_KEY" | base64 -d -o "$APPSTORECONNECT_API_KEY_PATH"
-          for file in syncthing-macos-*.zip ; do
-            xcrun notarytool submit \
-              -k "$APPSTORECONNECT_API_KEY_PATH" \
-              -d "$APPSTORECONNECT_API_KEY_ID" \
-              -i "$APPSTORECONNECT_API_KEY_ISSUER" \
-              $file
-          done
-        env:
-          APPSTORECONNECT_API_KEY: ${{ secrets.APPSTORECONNECT_API_KEY }}
-          APPSTORECONNECT_API_KEY_ID: ${{ secrets.APPSTORECONNECT_API_KEY_ID }}
-          APPSTORECONNECT_API_KEY_ISSUER: ${{ secrets.APPSTORECONNECT_API_KEY_ISSUER }}
-
-  #
-  # Cross compile other unixes
-  #
-
-  package-cross:
-    name: Package cross compiled
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-cross-${{ hashFiles('**/go.sum') }}
-
-      - name: Create packages
-        run: |
-          platforms=$(go tool dist list \
-            | grep -v aix/ppc64 \
-            | grep -v android/ \
-            | grep -v darwin/ \
-            | grep -v ios/ \
-            | grep -v js/ \
-            | grep -v linux/ \
-            | grep -v nacl/ \
-            | grep -v plan9/ \
-            | grep -v windows/ \
-            | grep -v /wasm \
-          )
-
-          # Build for each platform with errors silenced, because we expect
-          # some oddball platforms to fail. This avoids a bunch of errors in
-          # the GitHub Actions output, instead summarizing each build
-          # failure as a warning.
-          for plat in $platforms; do
-            goos="${plat%/*}"
-            goarch="${plat#*/}"
-            echo "::group ::$plat"
-            if ! go run build.go -goos "$goos" -goarch "$goarch" tar 2>/dev/null; then
-              echo "::warning ::Failed to build for $plat"
-            fi
-            echo "::endgroup::"
-          done
-        env:
-          CGO_ENABLED: "0"
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: packages-other
-          path: syncthing-*.tar.gz
-
-  #
-  # Source
-  #
-
-  package-source:
-    name: Package source code
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Package source
-        run: |
-          version=$(go run build.go version)
-          echo "$version" > RELEASE
-
-          go mod vendor
-          go run build.go assets
-
-          cd ..
-
-          tar c -z -f "syncthing-source-$version.tar.gz" \
-            --exclude .git \
-            syncthing
-
-          mv "syncthing-source-$version.tar.gz" syncthing
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: packages-source
-          path: syncthing-source-*.tar.gz
-
-  #
-  # Sign binaries for auto upgrade, generate ASC signature files
-  #
-
-  sign-for-upgrade:
-    name: Sign for upgrade
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
-    environment: signing
-    needs:
-      - basics
-      - package-windows
-      - package-linux
-      - package-macos
-      - package-cross
-      - package-source
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/checkout@v4
-        with:
-          repository: syncthing/release-tools
-          path: tools
-          fetch-depth: 0
-
-      - name: Download artifacts
-        uses: actions/download-artifact@v3
-
-      - name: Install signing tool
-        run: |
-          go install ./cmd/stsigtool
-
-      - name: Sign archives
-        run: |
-          export PRIVATE_KEY="$RUNNER_TEMP/privkey.pem"
-          export PATH="$PATH:$(go env GOPATH)/bin"
-          echo "$STSIGTOOL_PRIVATE_KEY" | base64 -d > "$PRIVATE_KEY"
-          mkdir packages
-          mv packages-*/* packages
-          pushd packages
-          "$GITHUB_WORKSPACE/tools/sign-only"
-          rm -f "$PRIVATE_KEY"
-        env:
-          STSIGTOOL_PRIVATE_KEY: ${{ secrets.STSIGTOOL_PRIVATE_KEY }}
-
-      - name: Create and sign .asc files
-        run: |
-          sudo apt update
-          sudo apt -y install gnupg
-
-          export SIGNING_KEY="$RUNNER_TEMP/gpg-secret.asc"
-          echo "$GNUPG_SIGNING_KEY_BASE64" | base64 -d > "$SIGNING_KEY"
-          gpg --import < "$SIGNING_KEY"
-
-          pushd packages
-          files=(*.tar.gz *.zip)
-          sha1sum "${files[@]}" | gpg --clearsign > sha1sum.txt.asc
-          sha256sum "${files[@]}" | gpg --clearsign > sha256sum.txt.asc
-          gpg --sign --armour --detach syncthing-source-*.tar.gz
-          popd
-          rm -f "$SIGNING_KEY" .gnupg
-        env:
-          GNUPG_SIGNING_KEY_BASE64: ${{ secrets.GNUPG_SIGNING_KEY_BASE64 }}
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: packages-signed
-          path: packages/*
-
-  #
-  # Debian
-  #
-
-  package-debian:
-    name: Package for Debian
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: '3.0'
-
-      - name: Install fpm
-        run: |
-          gem install fpm
-
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-debian-${{ hashFiles('**/go.sum') }}
-
-      - name: Package for Debian
-        run: |
-          for arch in amd64 i386 armhf armel arm64 ; do
-            go run build.go -no-upgrade -installsuffix=no-upgrade -goarch "$arch" deb
-          done
-        env:
-          BUILD_USER: debian
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: debian-packages
-          path: "*.deb"
-
-  #
-  # Nightlies
-  #
-
-  publish-nightly:
-    name: Publish nightly build
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && startsWith(github.ref, 'refs/heads/release-nightly')
-    environment: signing
-    needs:
-      - sign-for-upgrade
-      - notarize-macos
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: syncthing/release-tools
-          path: tools
-          fetch-depth: 0
-
-      - name: Download artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: packages-signed
-          path: packages
-
-      - name: Create release json
-        run: |
-          cd packages
-          "$GITHUB_WORKSPACE/tools/generate-release-json" "$BASE_URL" > nightly.json
-        env:
-          BASE_URL: https://syncthing.ams3.digitaloceanspaces.com/nightly/
-
-      - name: Push artifacts
-        uses: docker://docker.io/rclone/rclone:latest
-        env:
-          RCLONE_CONFIG_SPACES_TYPE: s3
-          RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
-          RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
-          RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
-          RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
-          RCLONE_CONFIG_SPACES_ACL: public-read
-        with:
-          args: sync packages spaces:syncthing/nightly
-
-  #
-  # Push release artifacts to Spaces
-  #
-
-  publish-release-files:
-    name: Publish release files
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/release'
-    environment: signing
-    needs:
-      - sign-for-upgrade
-      - package-debian
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Download signed packages
-        uses: actions/download-artifact@v3
-        with:
-          name: packages-signed
-          path: packages
-
-      - name: Download debian packages
-        uses: actions/download-artifact@v3
-        with:
-          name: debian-packages
-          path: packages
-
-      - name: Set version
-        run: |
-          version=$(go run build.go version)
-          echo "VERSION=$version" >> $GITHUB_ENV
-
-      - name: Push to Spaces (${{ env.VERSION }})
-        uses: docker://docker.io/rclone/rclone:latest
-        env:
-          RCLONE_CONFIG_SPACES_TYPE: s3
-          RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
-          RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
-          RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
-          RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
-          RCLONE_CONFIG_SPACES_ACL: public-read
-        with:
-          args: sync packages spaces:syncthing/release/${{ env.VERSION }}
-
-      - name: Push to Spaces (latest)
-        uses: docker://docker.io/rclone/rclone:latest
-        env:
-          RCLONE_CONFIG_SPACES_TYPE: s3
-          RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
-          RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
-          RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
-          RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
-          RCLONE_CONFIG_SPACES_ACL: public-read
-        with:
-          args: sync spaces:syncthing/release/${{ env.VERSION }} spaces:syncthing/release/latest
-
-  #
-  # Build and push to Docker Hub
-  #
-
-  docker-syncthing:
-    name: Build and push Docker images
-    runs-on: ubuntu-latest
-    if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release-'))
-    environment: docker
-    strategy:
-      matrix:
-        pkg:
-          - syncthing
-          - strelaysrv
-          - stdiscosrv
-        include:
-          - pkg: syncthing
-            dockerfile: Dockerfile
-            image: syncthing/syncthing
-          - pkg: strelaysrv
-            dockerfile: Dockerfile.strelaysrv
-            image: syncthing/relaysrv
-          - pkg: stdiscosrv
-            dockerfile: Dockerfile.stdiscosrv
-            image: syncthing/discosrv
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-docker-${{ matrix.pkg }}-${{ hashFiles('**/go.sum') }}
-
-      - name: Build binaries
-        run: |
-          for arch in amd64 arm64 arm; do
-            go run build.go -goos linux -goarch "$arch" -no-upgrade build ${{ matrix.pkg }}
-            mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-"$arch"
-          done
-        env:
-          CGO_ENABLED: "0"
-          BUILD_USER: docker
-
-      - name: Check if we will be able to push images
-        run: |
-          if [[ "${{ secrets.DOCKERHUB_TOKEN }}" != "" ]]; then
-            echo "DOCKER_PUSH=true" >> $GITHUB_ENV;
-          fi
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        if: env.DOCKER_PUSH == 'true'
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Set version tags
-        run: |
-          version=$(go run build.go version)
-          version=${version#v}
-          if [[ $version == @([0-9]|[0-9][0-9]).@([0-9]|[0-9][0-9]).@([0-9]|[0-9][0-9]) ]] ; then
-            echo Release version, pushing to :latest and version tags
-            major=${version%.*.*}
-            minor=${version%.*}
-            tags=${{ matrix.image }}:$version,${{ matrix.image }}:$major,${{ matrix.image }}:$minor,${{ matrix.image }}:latest
-          elif [[ $version == *-rc.@([0-9]|[0-9][0-9]) ]] ; then
-            echo Release candidate, pushing to :rc
-            tags=${{ matrix.image }}:rc
-          else
-            echo Development version, pushing to :edge
-            tags=${{ matrix.image }}:edge
-          fi
-          echo "DOCKER_TAGS=$tags" >> $GITHUB_ENV
-
-      - name: Build and push Docker image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ${{ matrix.dockerfile }}
-          platforms: linux/amd64,linux/arm64,linux/arm/7
-          push: ${{ env.DOCKER_PUSH == 'true' }}
-          tags: ${{ env.DOCKER_TAGS }}
-
-  #
-  # Check for known vulnerabilities in Go dependencies
-  #
-
-  govulncheck:
-    runs-on: ubuntu-latest
-    name: Run govulncheck
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: run govulncheck
-        run: |
-          go run build.go assets
-          go install golang.org/x/vuln/cmd/govulncheck@latest
-          govulncheck ./...

.github/workflows/trigger-nightly.yaml

@@ -1,21 +0,0 @@
-name: Trigger nightly build & release
-on:
-  workflow_dispatch:
-  schedule:
-    # Run nightly build at 01:00 UTC
-    - cron: '00 01 * * *'
-
-jobs:
-
-  trigger-nightly:
-    runs-on: ubuntu-latest
-    name: Push to release-nightly to trigger build
-    steps:
-
-      - uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
-          fetch-depth: 0
-
-      - run: |
-          git push origin main:release-nightly

.github/workflows/update-docs-translations.yaml

@@ -1,28 +0,0 @@
-name: Update translations and documentation
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: '42 3 * * 1'
-
-jobs:
-
-  update_transifex_docs:
-    runs-on: ubuntu-latest
-    name: Update translations and documentation
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ^1.19.6
-      - run: |
-          set -euo pipefail
-          git config --global user.name 'Syncthing Release Automation'
-          git config --global user.email 'release@syncthing.net'
-          bash build.sh translate
-          bash build.sh prerelease
-          git push
-        env:
-          WEBLATE_TOKEN: ${{ secrets.WEBLATE_TOKEN }}

.gitignore

@@ -1,10 +1,11 @@
 /syncthing
 /stdiscosrv
+syncthing.exe
+stdiscosrv.exe
 *.tar.gz
 *.zip
 *.asc
 *.deb
-*.exe
 .jshintrc
 coverage.out
 files/pidx
@@ -14,7 +15,12 @@ coverage.xml
 syncthing.sig
 RELEASE
 deb
+lib/auto/gui.files.go
+snapcraft.yaml
+prime/
+snap/
+parts/
+stage/
+*.snap
 *.bz2
 /repos
-/proto/scripts/protoc-gen-gosyncthing
-/gui/next-gen-gui

.golangci.yml

@@ -1,26 +0,0 @@
-linters-settings:
-  maligned:
-    suggest-new: true
-
-linters:
-  enable-all: true
-  disable:
-    - goimports
-    - depguard
-    - lll
-    - gochecknoinits
-    - gochecknoglobals
-    - gofmt
-    - scopelint
-    - gocyclo
-    - funlen
-    - wsl
-    - gocognit
-    - godox
-
-service:
-  golangci-lint-version: 1.21.x
-  prepare:
-    - rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
-    - GO111MODULE=on go mod vendor
-    - go run build.go assets

.yamlfmt

@@ -1,4 +0,0 @@
-line_ending: lf
-formatter:
-  type: basic
-  retain_line_breaks: true

AUTHORS

@@ -16,38 +16,20 @@
 Aaron Bieber (qbit) <qbit@deftly.net>
 Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com> <adam@proactiveservices.co.uk>
 Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
-Alan Pope <alan@popey.com>
-Alberto Donato <albertodonato@users.noreply.github.com>
-Aleksey Vasenev <margtu-fivt@ya.ru>
 Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
-Alex Lindeman <139387+aelindeman@users.noreply.github.com>
-Alex Xu <alex.hello71@gmail.com>
 Alexander Graf (alex2108) <register-github@alex-graf.de>
-Alexander Seiler <seileralex@gmail.com>
-Alexandre Alves <alexandrealvesdb.contact@gmail.com>
 Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
-Aman Gupta <aman@tmm1.net>
 Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
-Andreas Sommer <andreas.sommer87@googlemail.com>
 andresvia <andres.via@gmail.com>
 Andrew Dunham (andrew-d) <andrew@du.nham.ca>
-Andrew Meyer <andrewm.bpi@gmail.com>
 Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
 Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
-André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
 andyleap <andyleap@gmail.com>
-Anjan Momi <anjan@momi.ca>
-Anthony Goeckner <agoeckner@users.noreply.github.com>
 Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
 Antony Male (canton7) <antony.male@gmail.com>
-Anur <anurnomeru@163.com>
 Aranjedeath <Aranjedeath@users.noreply.github.com>
-Arkadiusz Tymiński <gevleeog@gmail.com>
-Aroun <login@b-vo.fr>
 Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
-Artur Zubilewicz <AkaZecik@users.noreply.github.com>
-Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
-Aurélien Rainone <476650+arl@users.noreply.github.com>
+Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com>
 BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
 Bart De Vries (mogwa1) <devriesb@gmail.com>
 Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
@@ -56,205 +38,112 @@ Ben Shepherd (benshep) <bjashepherd@gmail.com>
 Ben Sidhom (bsidhom) <bsidhom@gmail.com>
 Benedikt Heine (bebehei) <bebe@bebehei.de>
 Benedikt Morbach <benedikt.morbach@googlemail.com>
-Benjamin Nater <17193640+bn4t@users.noreply.github.com>
 Benno Fünfstück <benno.fuenfstueck@gmail.com>
 Benny Ng (tpng) <benny.tpng@gmail.com>
-boomsquared <54829195+boomsquared@users.noreply.github.com>
-Boqin Qin <bobbqqin@bupt.edu.cn>
 Boris Rybalkin <ribalkin@gmail.com>
 Brandon Philips (philips) <brandon@ifup.org>
 Brendan Long (brendanlong) <self@brendanlong.com>
 Brian R. Becker (brbecker) <brbecker@gmail.com>
-bt90 <btom1990@googlemail.com>
 Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
-Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
+Carsten Hagemann (Moter8) <moter8@gmail.com>
 Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
 Cedric Staniewski (xduugu) <cedric@gmx.ca>
-chenrui <rui@meetup.com>
-Chih-Hsuan Yen <yan12125@gmail.com> <1937689+yan12125@users.noreply.github.com>
-Choongkyu <choongkyu.kim+gh@gmail.com> <vapidlyrapid+gh@gmail.com>
 Chris Howie (cdhowie) <me@chrishowie.com>
 Chris Joel (cdata) <chris@scriptolo.gy>
 Chris Tonkinson <chris@masterbran.ch>
-Christian Kujau <ckujau@users.noreply.github.com>
-Christian Prescott <me@christianprescott.com>
 chucic <chucic@seznam.cz>
 Colin Kennedy (moshen) <moshen.colin@gmail.com>
-Cromefire_ <tim.l@nghorst.net> <26320625+cromefire@users.noreply.github.com>
-cui fliter <imcusg@gmail.com>
-Cyprien Devillez <cypx@users.noreply.github.com>
-d-volution <49024624+d-volution@users.noreply.github.com>
+Cromefire_ <tim.l@nghorst.net>
 Dale Visser <dale.visser@live.com>
-Dan <benda.daniel@gmail.com>
-Daniel Barczyk <46358936+DanielBarczyk@users.noreply.github.com>
 Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
 Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
 Daniel Martí (mvdan) <mvdan@mvdan.cc>
 Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
 David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
-deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com>
-DeflateAwning <11021263+DeflateAwning@users.noreply.github.com>
 Denis A. (dva) <denisva@gmail.com>
 Dennis Wilson (snnd) <dw@risu.io>
-dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com> <27856297+dependabot-preview[bot]@users.noreply.github.com>
-dependabot[bot] <dependabot[bot]@users.noreply.github.com> <49699333+dependabot[bot]@users.noreply.github.com>
 derekriemer <derek.riemer@colorado.edu>
 desbma <desbma@users.noreply.github.com>
-Devon G. Redekopp <devon@redekopp.com>
-digital <didev@dinid.net>
-Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com>
 Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
-Domenic Horner <domenic@tgxn.net>
 Dominik Heidler (asdil12) <dominik@heidler.eu>
 Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
 Elliot Huffman <thelich2@gmail.com>
 Emil Hessman (ceh) <emil@hessman.se>
-Emil Lundberg <emil@emlun.se>
-Eng Zer Jun <engzerjun@gmail.com>
-entity0xfe <109791748+entity0xfe@users.noreply.github.com> <entity0xfe@my.domain>
-Eric Lesiuta <elesiuta@gmail.com>
-Eric P <eric@kastelo.net>
 Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
-Evan Spensley <94762716+0evan@users.noreply.github.com>
-Evgeny Kuznetsov <evgeny@kuznetsov.md>
 Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
-Felix <53702818+f-eliks@users.noreply.github.com>
 Felix Ableitner (Nutomic) <me@nutomic.com>
-Felix Lampe <mail@flampe.de>
 Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
 Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
 Frank Isemann (fti7) <frank@isemann.name>
-Gahl Saraf <saraf.gahl@gmail.com> <gahl@raftt.io>
-georgespatton <georgespatton@users.noreply.github.com>
-ghjklw <malo@jaffre.info>
 Gilli Sigurdsson (gillisig) <gilli@vx.is>
-Gleb Sinyavskiy <zhulik.gleb@gmail.com>
 Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
-greatroar <61184462+greatroar@users.noreply.github.com>
-Greg <gco@jazzhaiku.com>
-guangwu <guoguangwu@magic-shield.com>
 Han Boetes <han@boetes.org>
-HansK-p <42314815+HansK-p@users.noreply.github.com>
 Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
 Heiko Zuerker (Smiley73) <heiko@zuerker.org>
 Hugo Locurcio <hugo.locurcio@hugo.pro>
 Iain Barnett <iainspeed@gmail.com>
 Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
-ignacy123 <ignacy.buczek@onet.pl>
-Ikko Ashimine <eltociear@gmail.com>
-Ilya Brin <464157+ilyabrin@users.noreply.github.com>
-Iskander Sharipov (Alex) <quasilyte@gmail.com>
 Jaakko Hannikainen (jgke) <jgke@jgke.fi>
 Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
-Jack Croft <jccroft1@users.noreply.github.com>
-Jacob <jyundt@gmail.com>
 Jake Peterson (acogdev) <jake@acogdev.com>
 Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
-James O'Beirne <wild-github@au92.org>
 James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
 janost <janost@tuta.io>
-Jaroslav Lichtblau <svetlemodry@users.noreply.github.com>
 Jaroslav Malec (dzarda) <dzardacz@gmail.com>
 jaseg <githubaccount@jaseg.net>
-Jauder Ho <jauderho@users.noreply.github.com>
 Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
-Jaya Kumar <jaya.kumar@ict.nl>
-Jeffery To <jeffery.to@gmail.com>
-jelle van der Waa <jelle@vdwaa.nl>
 Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
 Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
-Jesse Lucas <jesse@jesselucas.com>
 Jochen Voss (seehuhn) <voss@seehuhn.de>
 Johan Andersson <j@i19.se>
 Johan Vromans (sciurius) <jvromans@squirrel.nl>
 John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
-Jonas Thelemann <e-mail@jonas-thelemann.de>
-Jonathan <artback@protonmail.com> <jonagn@gmail.com>
 Jonathan Cross <jcross@gmail.com>
-Jonta <359397+Jonta@users.noreply.github.com>
 Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
-jtagcat <git-514635f7@jtag.cat> <git-12dbd862@jtag.cat>
 Jörg Thalheim <Mic92@users.noreply.github.com>
-Jędrzej Kula <kula.jedrek@gmail.com>
-K.B.Dharun Krishna <kbdharunkrishna@gmail.com>
-Kalle Laine <pahakalle@protonmail.com>
 Karol Różycki (krozycki) <rozycki.karol@gmail.com>
-Kebin Liu <lkebin@gmail.com>
-Keith Harrison <keithh@protonmail.com>
 Keith Turner <kturner@apache.org>
 Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
 Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
 Kevin Allen (ironmig) <kma1660@gmail.com>
-Kevin Bushiri (keevBush) <keevbush@gmail.com> <36192217+keevBush@users.noreply.github.com>
 Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
 klemens <ka7@github.com>
 Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
 Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
-Lars Lehtonen <lars.lehtonen@gmail.com>
 Laurent Arnoud <laurent@spkdev.net>
 Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
 Leo Arias (elopio) <yo@elopio.net>
 Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
 Lode Hoste (Zillode) <zillode@zillode.be>
 Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
-LSmithx2 <42276854+lsmithx2@users.noreply.github.com>
-Lukas Lihotzki <lukas@lihotzki.de>
-luzpaz <luzpaz@users.noreply.github.com>
 Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
 Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
 Marc Pujol (kilburn) <kilburn@la3.org>
 Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
 marco-m <marco.molteni@laposte.net>
-Marcus Legendre <marcus.legendre@gmail.com>
-Mario Majila <mariustshipichik@gmail.com>
 Mark Pulford (mpx) <mark@kyne.com.au>
-Martchus <martchus@gmx.net>
-Martin Polehla <p0l0us@users.noreply.github.com>
 Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
-Mateusz Ż <thedead4fun@live.com>
 Matic Potočnik <hairyfotr@gmail.com>
 Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
-Matt Robenolt <matt@ydekproductions.com>
 Matteo Ruina <matteo.ruina@gmail.com>
 Maurizio Tomasi <ziotom78@gmail.com>
-Max <github@germancoding.com>
 Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
 MaximAL <almaximal@ya.ru>
 Maxime Thirouin <m@moox.io>
-Maximilian <maxi.rostock@outlook.de>
-mclang <1721600+mclang@users.noreply.github.com>
 Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
 Michael Ploujnikov (plouj) <ploujj@gmail.com>
-Michael Rienstra <mrienstra@gmail.com>
 Michael Tilli (pyfisch) <pyfisch@gmail.com>
-MichaIng <micha@dietpi.com>
-Migelo <miha@filetki.si>
 Mike Boone <mike@boonedocks.net>
 MikeLund <MikeLund@users.noreply.github.com>
-MikolajTwarog <43782609+MikolajTwarog@users.noreply.github.com>
-Mingxuan Lin <gdlmx@users.noreply.github.com>
-mv1005 <49659413+mv1005@users.noreply.github.com>
 Nate Morrison (nrm21) <natemorrison@gmail.com>
-Naveen <172697+naveensrinivasan@users.noreply.github.com>
 Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
-Nick Busey <NickBusey@users.noreply.github.com>
 Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
 Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
-Nicolas Perraut <n.perraut@gmail.com>
 Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
 Nils Jakobi (thunderstorm99) <jakobi.nils@gmail.com>
-NinoM4ster <ninom4ster@gmail.com>
-Nitroretro <43112364+Nitroretro@users.noreply.github.com>
 NoLooseEnds <jon.koslung@gmail.com>
-Oliver Freyermuth <o.freyermuth@googlemail.com>
-orangekame3 <miya.org.0309@gmail.com>
-otbutz <tbutz@optitool.de>
-Otiel <Otiel@users.noreply.github.com>
-overkill <22098433+0verk1ll@users.noreply.github.com>
 Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
-Pablo <pbaeyens31+github@gmail.com>
 Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
-Paul Brit <paulbrit44@gmail.com>
 Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
 Paweł Rozlach <vespian@users.noreply.github.com>
 perewa <cavalcante.ten@gmail.com>
@@ -262,68 +151,45 @@ Peter Badida <KeyWeeUsr@users.noreply.github.com>
 Peter Dave Hello <hsu@peterdavehello.org>
 Peter Hoeg (peterhoeg) <peter@speartail.com>
 Peter Marquardt (wwwutz) <wwwutz@gmail.com> <wwwutz@googlemail.com>
-Phani Rithvij <phanirithvij2000@gmail.com>
 Phil Davis <phil.davis@inf.org>
 Philippe Schommers (filoozoom) <philippe@schommers.be>
 Phill Luby (pluby) <phill.luby@newredo.com>
 Pier Paolo Ramon <ramonpierre@gmail.com>
 Piotr Bejda (piobpl) <piotrb10@gmail.com>
 Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
-Quentin Hibon <qh.public@yahoo.com>
-Rahmi Pruitt <rjpruitt16@gmail.com>
-red_led <red-led@users.noreply.github.com>
 Richard Hartmann <RichiH@users.noreply.github.com>
 Robert Carosi (nov1n) <robert@carosi.nl>
-Roberto Santalla <roobre@users.noreply.github.com>
-Robin Schoonover <robin@cornhooves.org>
 Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
 Ross Smith II (rasa) <ross@smithii.com>
 rubenbe <github-com-00ff86@vandamme.email>
-Ruslan Yevdokymov <38809160+ruslanye@users.noreply.github.com>
-Ryan Qian <i@bitbili.net>
 Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
 Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
 Scott Klupfel (kluppy) <kluppy@going2blue.com>
-sec65 <106604020+sec65@users.noreply.github.com>
 Sergey Mishin (ralder) <ralder@yandex.ru>
-Shaarad Dalvi <60266155+shaaraddalvi@users.noreply.github.com> <shdalv@microsoft.com>
 Simon Frei (imsodin) <freisim93@gmail.com>
-Simon Mwepu <simonmwepu@gmail.com>
 Sly_tom_cat <slytomcat@mail.ru>
 Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
 Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
-Steven Eckhoff <steven.eckhoff.opensource@gmail.com>
 Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
-Syncthing Automation <automation@syncthing.net>
-Syncthing Release Automation <release@syncthing.net>
 Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
 Thomas Hipp <thomashipp@gmail.com>
 Tim Abell (timabell) <tim@timwise.co.uk>
 Tim Howes (timhowes) <timhowes@berkeley.edu>
-Tobias Klauser <tobias.klauser@gmail.com>
 Tobias Nygren (tnn2) <tnn@nygren.pp.se>
 Tobias Tom (tobiastom) <t.tom@succont.de>
-Tom Jakubowski <tom@crystae.net>
-Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com> <twilczynski@naver.com>
+Tomas Cerveny (kozec) <kozec@kozec.com>
 Tommy Thorn <tommy-github-email@thorn.ws>
 Tully Robinson (tojrobinson) <tully@tojr.org>
 Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
-Tyler Kropp <kropptyler@gmail.com>
 Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
 Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
 Victor Buinsky (buinsky) <vix_booja@tut.by>
-Vik <63919734+ViktorOn@users.noreply.github.com>
 Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
-villekalliomaki <53118179+villekalliomaki@users.noreply.github.com>
-Vladimir Rusinov <vrusinov@google.com> <vladimir.rusinov@gmail.com>
+Vladimir Rusinov <vrusinov@google.com>
 wangguoliang <liangcszzu@163.com>
-Will Rouesnel <wrouesnel@wrouesnel.com>
 William A. Kennington III (wkennington) <william@wkennington.com>
-wouter bolsterlee <wouter@bolsterl.ee>
 Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
-xarx00 <xarx00@users.noreply.github.com>
 Xavier O. (damajor) <damajor@gmail.com>
 xjtdy888 (xjtdy888) <xjtdy888@163.com>
 Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
 佛跳墙 <daoquan@qq.com>
-落心 <luoxin.ttt@gmail.com>

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 ## Reporting Bugs
 
-Please file bugs in the [GitHub Issue
+Please file bugs in the [Github Issue
 Tracker](https://github.com/syncthing/syncthing/issues). Include at
 least the following:
 
@@ -24,15 +24,10 @@ too much information will never get you yelled at. :)
 ## Contributing Translations
 
 All translations are done via
-[Weblate](https://hosted.weblate.org/projects/syncthing/). If you wish
-to contribute to a translation, just head over there and sign up.
+[Transifex](https://www.transifex.com/projects/p/syncthing/). If you
+wish to contribute to a translation, just head over there and sign up.
 Before every release, the language resources are updated from the
-latest info on Weblate.
-
-Note that the previously used service at
-[Transifex](https://www.transifex.com/projects/p/syncthing/) is being
-retired and we kindly ask you to sign up on Weblate for continued
-involvement.
+latest info on Transifex.
 
 ## Contributing Code
 

Dockerfile

@@ -1,49 +1,32 @@
-ARG GOVERSION=latest
+FROM golang:1.11 AS builder
 
-#
-# Maybe build Syncthing. This is a bit ugly as we can't make an entire
-# section of the Dockerfile conditional, so we end up always pulling the
-# golang image as builder. Then we check if the executable we need already
-# exists (pre-built) otherwise we build it.
-#
-
-FROM golang:$GOVERSION AS builder
-ARG BUILD_USER
-ARG BUILD_HOST
-ARG TARGETARCH
-
-WORKDIR /src
+WORKDIR /go/src/github.com/syncthing/syncthing
 COPY . .
 
 ENV CGO_ENABLED=0
-RUN if [ ! -f syncthing-linux-$TARGETARCH ] ; then \
-    go run build.go -no-upgrade build syncthing ; \
-    mv syncthing syncthing-linux-$TARGETARCH ; \
-  fi
-
-#
-# The rest of the Dockerfile uses the binary from the builder, prebuilt or
-# not.
-#
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f syncthing && go run build.go -no-upgrade build syncthing
 
 FROM alpine
-ARG TARGETARCH
 
-EXPOSE 8384 22000/tcp 22000/udp 21027/udp
+EXPOSE 8384 22000 21027/udp
 
 VOLUME ["/var/syncthing"]
 
-RUN apk add --no-cache ca-certificates curl libcap su-exec tzdata
+RUN apk add --no-cache ca-certificates su-exec
 
-COPY --from=builder /src/syncthing-linux-$TARGETARCH /bin/syncthing
-COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+COPY --from=builder /go/src/github.com/syncthing/syncthing/syncthing /bin/syncthing
 
-ENV PUID=1000 PGID=1000 HOME=/var/syncthing
+ENV PUID=1000 PGID=1000
 
 HEALTHCHECK --interval=1m --timeout=10s \
-  CMD curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -o --color=never OK || exit 1
+  CMD nc -z localhost 8384 || exit 1
 
-ENV STGUIADDRESS=0.0.0.0:8384
-ENV STHOMEDIR=/var/syncthing/config
-RUN chmod 755 /bin/entrypoint.sh
-ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing"]
+ENTRYPOINT \
+  chown "${PUID}:${PGID}" /var/syncthing \
+  && su-exec "${PUID}:${PGID}" \
+     env HOME=/var/syncthing \
+     /bin/syncthing \
+       -home /var/syncthing/config \
+       -gui-address 0.0.0.0:8384

Dockerfile.builder

@@ -1,9 +0,0 @@
-ARG GOVERSION=latest
-FROM golang:$GOVERSION
-
-# FPM to build Debian packages
-RUN apt-get update && apt-get install -y --no-install-recommends \
-	locales rubygems ruby-dev build-essential git \
-	&& apt-get clean \
-	&& rm -rf /var/lib/apt/lists/* \
-	&& gem install fpm

Dockerfile.stcrashreceiver

@@ -1,8 +0,0 @@
-FROM alpine
-ARG TARGETARCH
-
-EXPOSE 8080
-
-COPY stcrashreceiver-linux-${TARGETARCH} /bin/stcrashreceiver
-
-ENTRYPOINT [ "/bin/stcrashreceiver" ]

Dockerfile.stdiscosrv

@@ -1,34 +0,0 @@
-ARG GOVERSION=latest
-FROM golang:$GOVERSION AS builder
-ARG BUILD_USER
-ARG BUILD_HOST
-ARG TARGETARCH
-
-WORKDIR /src
-COPY . .
-
-ENV CGO_ENABLED=0
-RUN if [ ! -f stdiscosrv-linux-$TARGETARCH ] ; then \
-    go run build.go -no-upgrade build stdiscosrv ; \
-    mv stdiscosrv stdiscosrv-linux-$TARGETARCH ; \
-  fi
-
-FROM alpine
-ARG TARGETARCH
-
-EXPOSE 19200 8443
-
-VOLUME ["/var/stdiscosrv"]
-
-RUN apk add --no-cache ca-certificates su-exec
-
-COPY --from=builder /src/stdiscosrv-linux-$TARGETARCH /bin/stdiscosrv
-COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
-
-ENV PUID=1000 PGID=1000 HOME=/var/stdiscosrv
-
-HEALTHCHECK --interval=1m --timeout=10s \
-  CMD nc -z localhost 8443 || exit 1
-
-WORKDIR /var/stdiscosrv
-ENTRYPOINT ["/bin/entrypoint.sh", "/bin/stdiscosrv"]

Dockerfile.strelaypoolsrv

@@ -1,16 +0,0 @@
-FROM alpine
-ARG TARGETARCH
-
-EXPOSE 8080
-
-RUN apk add --no-cache ca-certificates su-exec curl
-ENV PUID=1000 PGID=1000 MAXMIND_KEY=
-
-RUN mkdir /var/strelaypoolsrv && chown 1000 /var/strelaypoolsrv
-USER 1000
-
-COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv
-COPY script/strelaypoolsrv-entrypoint.sh /bin/entrypoint.sh
-
-WORKDIR /var/strelaypoolsrv
-ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaypoolsrv", "-listen", ":8080"]

Dockerfile.strelaysrv

@@ -1,34 +0,0 @@
-ARG GOVERSION=latest
-FROM golang:$GOVERSION AS builder
-ARG BUILD_USER
-ARG BUILD_HOST
-ARG TARGETARCH
-
-WORKDIR /src
-COPY . .
-
-ENV CGO_ENABLED=0
-RUN if [ ! -f strelaysrv-linux-$TARGETARCH ] ; then \
-    go run build.go -no-upgrade build strelaysrv ; \
-    mv strelaysrv strelaysrv-linux-$TARGETARCH ; \
-  fi
-
-FROM alpine
-ARG TARGETARCH
-
-EXPOSE 22067 22070
-
-VOLUME ["/var/strelaysrv"]
-
-RUN apk add --no-cache ca-certificates su-exec
-
-COPY --from=builder /src/strelaysrv-linux-$TARGETARCH /bin/strelaysrv
-COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
-
-ENV PUID=1000 PGID=1000 HOME=/var/strelaysrv
-
-HEALTHCHECK --interval=1m --timeout=10s \
-  CMD nc -z localhost 22067 || exit 1
-
-WORKDIR /var/strelaysrv
-ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaysrv"]

Dockerfile.stupgrades

@@ -1,8 +0,0 @@
-FROM alpine
-ARG TARGETARCH
-
-EXPOSE 8080
-
-COPY stupgrades-linux-${TARGETARCH} /bin/stupgrades
-
-ENTRYPOINT [ "/bin/stupgrades" ]

GOALS.md

@@ -24,17 +24,17 @@ to avoid corrupting the user's files.
 ### 2. Secure Against Attackers
 
 Again, protecting the user's data is paramount. Regardless of our other
-goals, we must never allow the user's data to be susceptible to eavesdropping
+goals we must never allow the user's data to be susceptible to eavesdropping
 or modification by unauthorized parties.
 
 > This should be understood in context. It is not necessarily reasonable to
 > expect Syncthing to be resistant against well equipped state level
-> attackers. We will, however, do our best. Note also that this is different
+> attackers. We will however do our best. Note also that this is different
 > from anonymity which is not, currently, a goal.
 
 ### 3. Easy to Use
 
-Syncthing should be approachable, understandable, and inclusive.
+Syncthing should be approachable, understandable and inclusive.
 
 > Complex concepts and maths form the base of Syncthing's functionality.
 > This should nonetheless be abstracted or hidden to a degree where
@@ -52,18 +52,18 @@ User interaction should be required only when absolutely necessary.
 ### 5. Universally Available
 
 Syncthing should run on every common computer. We are mindful that the
-latest technology is not always available to every individual.
+latest technology is not always available to any given individual.
 
 > Computers include desktops, laptops, servers, virtual machines, small
 > general purpose computers such as Raspberry Pis and, *where possible*,
-> tablets and phones. NAS appliances, toasters, cars, firearms, thermostats,
-> and so on may include computing capabilities but it is not our goal for
+> tablets and phones. NAS appliances, toasters, cars, firearms, thermostats
+> and so on may include computing capabitilies but it is not our goal for
 > Syncthing to run smoothly on these devices.
 
 ### 6. For Individuals
 
 Syncthing is primarily about empowering the individual user with safe,
-secure, and easy to use file synchronization.
+secure and easy to use file synchronization.
 
 > We acknowledge that it's also useful in an enterprise setting and include
 > functionality to support that. If this is in conflict with the

README-Docker.md

@@ -1,109 +1,43 @@
 # Docker Container for Syncthing
 
 Use the Dockerfile in this repo, or pull the `syncthing/syncthing` image
-from Docker Hub.
+from Docker Hub. Use volumes to have the synchronized files available on the
+host.
 
-Use the `/var/syncthing` volume to have the synchronized files available on the
-host. You can add more folders and map them as you prefer.
+The exposed volumes are by default:
+
+    /var/syncthing/config   - the configuration and index directory into the Container
+    /var/syncthing          - the default sync folder into the Container
+
+You can add more folders and map them as you prefer.
 
 Note that Syncthing runs as UID 1000 and GID 1000 by default. These may be
-altered with the `PUID` and `PGID` environment variables. In addition
-the name of the Syncthing instance can be optionally defined by using
-`--hostname=syncthing` parameter.
+altered with the ``PUID`` and ``PGID`` environment variables.
 
-To grant Syncthing additional capabilities without running as root, use the
-`PCAP` environment variable with the same syntax as that for `setcap(8)`.
-For example, `PCAP=cap_chown,cap_fowner+ep`.
+Example usage:
 
-## Example Usage
-
-**Docker cli**
 ```
 $ docker pull syncthing/syncthing
-$ docker run -p 8384:8384 -p 22000:22000/tcp -p 22000:22000/udp -p 21027:21027/udp \
+$ docker run -p 8384:8384 -p 22000:22000 \
+    -v /wherever/st-cfg:/var/syncthing/config \
     -v /wherever/st-sync:/var/syncthing \
-    --hostname=my-syncthing \
     syncthing/syncthing:latest
 ```
 
-**Docker compose**
-```yml
----
-version: "3"
-services:
-  syncthing:
-    image: syncthing/syncthing
-    container_name: syncthing
-    hostname: my-syncthing
-    environment:
-      - PUID=1000
-      - PGID=1000
-    volumes:
-      - /wherever/st-sync:/var/syncthing
-    ports:
-      - 8384:8384 # Web UI
-      - 22000:22000/tcp # TCP file transfers
-      - 22000:22000/udp # QUIC file transfers
-      - 21027:21027/udp # Receive local discovery broadcasts
-    restart: unless-stopped
-```
+Note that local device discovery will not work with the above command resulting
+in poor local transfer rates if local device addresses are not manually
+configured.
 
-## Discovery
+To allow local discovery, the docker host network can be used instead:
 
-Note that Docker's default network mode prevents local IP addresses from
-being discovered, as Syncthing is only able to see the internal IP of the
-container on the `172.17.0.0/16` subnet. This will result in poor transfer rates
-if local device addresses are not manually configured.
-
-It is therefore advisable to use the [host network mode](https://docs.docker.com/network/host/) instead:
-
-**Docker cli**
 ```
 $ docker pull syncthing/syncthing
 $ docker run --network=host \
+    -v /wherever/st-cfg:/var/syncthing/config \
     -v /wherever/st-sync:/var/syncthing \
     syncthing/syncthing:latest
 ```
 
-**Docker compose**
-```yml
----
-version: "3"
-services:
-  syncthing:
-    image: syncthing/syncthing
-    container_name: syncthing
-    hostname: my-syncthing
-    environment:
-      - PUID=1000
-      - PGID=1000
-    volumes:
-      - /wherever/st-sync:/var/syncthing
-    network_mode: host
-    restart: unless-stopped
-```
-
 Be aware that syncthing alone is now in control of what interfaces and ports it
 listens on. You can edit the syncthing configuration to change the defaults if
 there are conflicts.
-
-## GUI Security
-
-By default Syncthing inside the Docker image listens on 0.0.0.0:8384 to
-allow GUI connections via the Docker proxy. This is set by the
-`STGUIADDRESS` environment variable in the Dockerfile, as it differs from
-what Syncthing would otherwise use by default. This means you should set up
-authentication in the GUI, like for any other externally reachable Syncthing
-instance. If you do not require the GUI, or you use host networking, you can
-unset the `STGUIADDRESS` variable to have Syncthing fall back to listening
-on 127.0.0.1:
-
-```
-$ docker pull syncthing/syncthing
-$ docker run -e STGUIADDRESS= \
-    -v /wherever/st-sync:/var/syncthing \
-    syncthing/syncthing:latest
-```
-
-With the environment variable unset Syncthing will follow what is set in the
-configuration file / GUI settings dialog.

README.md

@@ -2,6 +2,10 @@
 
 ---
 
+[![Latest Downloads](https://img.shields.io/badge/latest-downloads-brightgreen.svg?style=flat-square)](https://build.syncthing.net/latest/)
+[![Latest Linux & Cross Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildLinuxCross.svg?style=flat-square&label=linux+%26+cross+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildLinuxCross&guest=1)
+[![Latest Windows Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildWindows.svg?style=flat-square&label=windows+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildWindows&guest=1)
+[![Latest Mac Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildMac.svg?style=flat-square&label=mac+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildMac&guest=1)
 [![MPLv2 License](https://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/88/badge)](https://bestpractices.coreinfrastructure.org/projects/88)
 [![Go Report Card](https://goreportcard.com/badge/github.com/syncthing/syncthing)](https://goreportcard.com/report/github.com/syncthing/syncthing)
@@ -10,8 +14,8 @@
 
 Syncthing is a **continuous file synchronization program**. It synchronizes
 files between two or more computers. We strive to fulfill the goals below.
-The goals are listed in order of importance, the most important ones first.
-This is the summary version of the goal list - for more
+The goals are listed in order of importance, the most important one being
+the first. This is the summary version of the goal list - for more
 commentary, see the full [Goals document][13].
 
 Syncthing should be:
@@ -24,12 +28,12 @@ Syncthing should be:
 2. **Secure Against Attackers**
 
    Again, protecting the user's data is paramount. Regardless of our other
-   goals, we must never allow the user's data to be susceptible to
+   goals we must never allow the user's data to be susceptible to
    eavesdropping or modification by unauthorized parties.
 
 3. **Easy to Use**
 
-   Syncthing should be approachable, understandable, and inclusive.
+   Syncthing should be approachable, understandable and inclusive.
 
 4. **Automatic**
 
@@ -38,12 +42,12 @@ Syncthing should be:
 5. **Universally Available**
 
    Syncthing should run on every common computer. We are mindful that the
-   latest technology is not always available to every individual.
+   latest technology is not always available to any given individual.
 
 6. **For Individuals**
 
    Syncthing is primarily about empowering the individual user with safe,
-   secure, and easy to use file synchronization.
+   secure and easy to use file synchronization.
 
 7. **Everything Else**
 
@@ -57,63 +61,54 @@ Take a look at the [getting started guide][2].
 
 There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory][3]. There are also several [GUI
-implementations][11] for Windows, Mac, and Linux.
-
-## Docker
-
-To run Syncthing in Docker, see [the Docker README][16].
+implementations][11] for Windows, Mac and Linux.
 
 ## Vote on features/bugs
 
 We'd like to encourage you to [vote][12] on issues that matter to you.
-This helps the team understand what are the biggest pain points for our
-users, and could potentially influence what is being worked on next.
+This helps the team understand what are the biggest pain points for our users, and could potentially influence what is being worked on next.
 
 ## Getting in Touch
 
-The first and best point of contact is the [Forum][8].
-If you've found something that is clearly a
+The first and best point of contact is the [Forum][8]. There is also an IRC
+channel, `#syncthing` on [freenode][4] (with a [web client][9]), for talking
+directly to developers and users. If you've found something that is clearly a
 bug, feel free to report it in the [GitHub issue tracker][10].
 
-If you believe that you’ve found a Syncthing-related security vulnerability,
-please report it by emailing security@syncthing.net. Do not report it in the
-Forum or issue tracker.
-
 ## Building
 
-Building Syncthing from source is easy. After extracting the source bundle from
-a release or checking out git, you just need to run `go run build.go` and the
-binaries are created in `./bin`. There's [a guide][5] with more details on the
-build process.
+Building Syncthing from source is easy, and there's a [guide][5]
+that describes it for both Unix and Windows systems.
 
 ## Signed Releases
 
-As of v0.10.15 and onwards, release binaries are GPG signed with the key
+As of v0.10.15 and onwards release binaries are GPG signed with the key
 D26E6ED000654A3E, available from https://syncthing.net/security.html and
 most key servers.
 
-There is also a built-in automatic upgrade mechanism (disabled in some
+There is also a built in automatic upgrade mechanism (disabled in some
 distribution channels) which uses a compiled in ECDSA signature. macOS
 binaries are also properly code signed.
 
 ## Documentation
 
-Please see the Syncthing [documentation site][6] [[source]][17].
+Please see the [Syncthing documentation site][6].
 
 All code is licensed under the [MPLv2 License][7].
 
 [1]: https://docs.syncthing.net/specs/bep-v1.html
 [2]: https://docs.syncthing.net/intro/getting-started.html
-[3]: https://github.com/syncthing/syncthing/blob/main/etc
+[3]: https://github.com/syncthing/syncthing/blob/master/etc
+[4]: https://www.freenode.net/
 [5]: https://docs.syncthing.net/dev/building.html
 [6]: https://docs.syncthing.net/
-[7]: https://github.com/syncthing/syncthing/blob/main/LICENSE
+[7]: https://github.com/syncthing/syncthing/blob/master/LICENSE
 [8]: https://forum.syncthing.net/
+[9]: https://kiwiirc.com/client/irc.freenode.net/#syncthing
 [10]: https://github.com/syncthing/syncthing/issues
 [11]: https://docs.syncthing.net/users/contrib.html#gui-wrappers
 [12]: https://www.bountysource.com/teams/syncthing/issues
-[13]: https://github.com/syncthing/syncthing/blob/main/GOALS.md
+[13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
 [14]: assets/logo-text-128.png
 [15]: https://syncthing.net/
-[16]: https://github.com/syncthing/syncthing/blob/main/README-Docker.md
-[17]: https://github.com/syncthing/docs
+

build.ps1

@@ -1,20 +0,0 @@
-function build {
-    go run build.go @args
-}
-
-$cmd, $rest = $args
-switch ($cmd) {
-    "test" {
-        $env:LOGGER_DISCARD=1
-        build test
-    }
-
-    "bench" {
-        $env:LOGGER_DISCARD=1
-        build bench
-    }
-
-    default {
-        build @rest
-    }
-}

build.sh

@@ -2,6 +2,8 @@
 set -euo pipefail
 IFS=$'\n\t'
 
+STTRACE=${STTRACE:-}
+
 script() {
 	name="$1"
 	shift
@@ -13,23 +15,88 @@ build() {
 }
 
 case "${1:-default}" in
+	default)
+		build
+		;;
+
+	clean)
+		build "$@"
+		;;
+
+	tar)
+		build "$@"
+		;;
+
+	assets)
+		build "$@"
+		;;
+
+	xdr)
+		build "$@"
+		;;
+
+	translate)
+		build "$@"
+		;;
+
+	deb)
+		build "$@"
+		;;
+
+	setup)
+		build "$@"
+		;;
+
 	test)
 		LOGGER_DISCARD=1 build test
 		;;
 
 	bench)
-		LOGGER_DISCARD=1 build bench
+		LOGGER_DISCARD=1 build bench | script benchfilter
 		;;
 
 	prerelease)
-		script authors
-		build weblate
+		go run script/authors.go
+		build transifex
 		pushd man ; ./refresh.sh ; popd
 		git add -A gui man AUTHORS
 		git commit -m 'gui, man, authors: Update docs, translations, and contributors'
 		;;
 
+	noupgrade)
+		build -no-upgrade tar
+		;;
+
+	all)
+		platforms=(
+			darwin-amd64 dragonfly-amd64 freebsd-amd64 linux-amd64 netbsd-amd64 openbsd-amd64 solaris-amd64 windows-amd64
+			freebsd-386 linux-386 netbsd-386 openbsd-386 windows-386
+			linux-arm linux-arm64 linux-ppc64 linux-ppc64le
+		)
+
+		for plat in "${platforms[@]}"; do
+			echo Building "$plat"
+
+			goos="${plat%-*}"
+			goarch="${plat#*-}"
+			dist="tar"
+
+			if [[ $goos == "windows" ]]; then
+				dist="zip"
+			fi
+
+			build -goos "$goos" -goarch "$goarch" "$dist"
+			echo
+		done
+		;;
+
+	test-xunit)
+
+		(GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -v -race ./lib/... ./cmd/... || true) > tests.out
+		go2xunit -output tests.xml -fail < tests.out
+		;;
+
 	*)
-		build "$@"
+		echo "Unknown build command $1"
 		;;
 esac

cmd/stbench/main.go

@@ -0,0 +1,143 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// This doesn't build on Windows due to the Rusage stuff.
+
+// +build !windows
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"runtime"
+	"syscall"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/rc"
+)
+
+var homeDir = "h1"
+var syncthingBin = "./bin/syncthing"
+var test = "scan"
+
+func main() {
+	flag.StringVar(&homeDir, "home", homeDir, "Home directory location")
+	flag.StringVar(&syncthingBin, "bin", syncthingBin, "Binary location")
+	flag.StringVar(&test, "test", test, "Test to run")
+	flag.Parse()
+
+	switch test {
+	case "scan":
+		// scan measures the resource usage required to perform the initial
+		// scan, without cleaning away the database first.
+		testScan()
+	}
+}
+
+// testScan starts a process and reports on the resource usage required to
+// perform the initial scan.
+func testScan() {
+	log.Println("Starting...")
+	p := rc.NewProcess("127.0.0.1:8081")
+	if err := p.Start(syncthingBin, "-home", homeDir, "-no-browser"); err != nil {
+		log.Println(err)
+		return
+	}
+	defer p.Stop()
+
+	wallTime := awaitScanComplete(p)
+
+	report(p, wallTime)
+}
+
+// awaitScanComplete waits for a folder to transition idle->scanning and
+// then scanning->idle and returns the time taken for the scan.
+func awaitScanComplete(p *rc.Process) time.Duration {
+	log.Println("Awaiting scan completion...")
+	var t0, t1 time.Time
+	lastEvent := 0
+loop:
+	for {
+		evs, err := p.Events(lastEvent)
+		if err != nil {
+			continue
+		}
+
+		for _, ev := range evs {
+			if ev.Type == "StateChanged" {
+				data := ev.Data.(map[string]interface{})
+				log.Println(ev)
+
+				if data["to"].(string) == "scanning" {
+					t0 = ev.Time
+					continue
+				}
+
+				if !t0.IsZero() && data["to"].(string) == "idle" {
+					t1 = ev.Time
+					break loop
+				}
+			}
+			lastEvent = ev.ID
+		}
+
+		time.Sleep(250 * time.Millisecond)
+	}
+
+	return t1.Sub(t0)
+}
+
+// report stops the given process and reports on its resource usage in two
+// ways: human readable to stderr, and CSV to stdout.
+func report(p *rc.Process, wallTime time.Duration) {
+	sv, err := p.SystemVersion()
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	ss, err := p.SystemStatus()
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	proc, err := p.Stop()
+	if err != nil {
+		return
+	}
+
+	rusage, ok := proc.SysUsage().(*syscall.Rusage)
+	if !ok {
+		return
+	}
+
+	log.Println("Version:", sv.Version)
+	log.Println("Alloc:", ss.Alloc/1024, "KiB")
+	log.Println("Sys:", ss.Sys/1024, "KiB")
+	log.Println("Goroutines:", ss.Goroutines)
+	log.Println("Wall time:", wallTime)
+	log.Println("Utime:", time.Duration(rusage.Utime.Nano()))
+	log.Println("Stime:", time.Duration(rusage.Stime.Nano()))
+	if runtime.GOOS == "darwin" {
+		// Darwin reports in bytes, Linux seems to report in KiB even
+		// though the manpage says otherwise.
+		rusage.Maxrss /= 1024
+	}
+	log.Println("MaxRSS:", rusage.Maxrss, "KiB")
+
+	fmt.Printf("%s,%d,%d,%d,%.02f,%.02f,%.02f,%d\n",
+		sv.Version,
+		ss.Alloc/1024,
+		ss.Sys/1024,
+		ss.Goroutines,
+		wallTime.Seconds(),
+		time.Duration(rusage.Utime.Nano()).Seconds(),
+		time.Duration(rusage.Stime.Nano()).Seconds(),
+		rusage.Maxrss)
+}

cmd/stcli/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2014 Audrius Butkevičius
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+- The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cmd/stcli/client.go

@@ -0,0 +1,115 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"net/http"
+	"strings"
+
+	"github.com/AudriusButkevicius/cli"
+)
+
+type APIClient struct {
+	httpClient http.Client
+	endpoint   string
+	apikey     string
+	username   string
+	password   string
+	id         string
+	csrf       string
+}
+
+var instance *APIClient
+
+func getClient(c *cli.Context) *APIClient {
+	if instance != nil {
+		return instance
+	}
+	endpoint := c.GlobalString("endpoint")
+	if !strings.HasPrefix(endpoint, "http") {
+		endpoint = "http://" + endpoint
+	}
+	httpClient := http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: c.GlobalBool("insecure"),
+			},
+		},
+	}
+	client := APIClient{
+		httpClient: httpClient,
+		endpoint:   endpoint,
+		apikey:     c.GlobalString("apikey"),
+		username:   c.GlobalString("username"),
+		password:   c.GlobalString("password"),
+	}
+
+	if client.apikey == "" {
+		request, err := http.NewRequest("GET", client.endpoint, nil)
+		die(err)
+		response := client.handleRequest(request)
+		client.id = response.Header.Get("X-Syncthing-ID")
+		if client.id == "" {
+			die("Failed to get device ID")
+		}
+		for _, item := range response.Cookies() {
+			if item.Name == "CSRF-Token-"+client.id[:5] {
+				client.csrf = item.Value
+				goto csrffound
+			}
+		}
+		die("Failed to get CSRF token")
+	csrffound:
+	}
+	instance = &client
+	return &client
+}
+
+func (client *APIClient) handleRequest(request *http.Request) *http.Response {
+	if client.apikey != "" {
+		request.Header.Set("X-API-Key", client.apikey)
+	}
+	if client.username != "" || client.password != "" {
+		request.SetBasicAuth(client.username, client.password)
+	}
+	if client.csrf != "" {
+		request.Header.Set("X-CSRF-Token-"+client.id[:5], client.csrf)
+	}
+
+	response, err := client.httpClient.Do(request)
+	die(err)
+
+	if response.StatusCode == 404 {
+		die("Invalid endpoint or API call")
+	} else if response.StatusCode == 401 {
+		die("Invalid username or password")
+	} else if response.StatusCode == 403 {
+		if client.apikey == "" {
+			die("Invalid CSRF token")
+		}
+		die("Invalid API key")
+	} else if response.StatusCode != 200 {
+		body := strings.TrimSpace(string(responseToBArray(response)))
+		if body != "" {
+			die(body)
+		}
+		die("Unknown HTTP status returned: " + response.Status)
+	}
+	return response
+}
+
+func httpGet(c *cli.Context, url string) *http.Response {
+	client := getClient(c)
+	request, err := http.NewRequest("GET", client.endpoint+"/rest/"+url, nil)
+	die(err)
+	return client.handleRequest(request)
+}
+
+func httpPost(c *cli.Context, url string, body string) *http.Response {
+	client := getClient(c)
+	request, err := http.NewRequest("POST", client.endpoint+"/rest/"+url, bytes.NewBufferString(body))
+	die(err)
+	return client.handleRequest(request)
+}

cmd/stcli/cmd_devices.go

@@ -0,0 +1,188 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/AudriusButkevicius/cli"
+	"github.com/syncthing/syncthing/lib/config"
+)
+
+func init() {
+	cliCommands = append(cliCommands, cli.Command{
+		Name:     "devices",
+		HideHelp: true,
+		Usage:    "Device command group",
+		Subcommands: []cli.Command{
+			{
+				Name:     "list",
+				Usage:    "List registered devices",
+				Requires: &cli.Requires{},
+				Action:   devicesList,
+			},
+			{
+				Name:     "add",
+				Usage:    "Add a new device",
+				Requires: &cli.Requires{"device id", "device name?"},
+				Action:   devicesAdd,
+			},
+			{
+				Name:     "remove",
+				Usage:    "Remove an existing device",
+				Requires: &cli.Requires{"device id"},
+				Action:   devicesRemove,
+			},
+			{
+				Name:     "get",
+				Usage:    "Get a property of a device",
+				Requires: &cli.Requires{"device id", "property"},
+				Action:   devicesGet,
+			},
+			{
+				Name:     "set",
+				Usage:    "Set a property of a device",
+				Requires: &cli.Requires{"device id", "property", "value..."},
+				Action:   devicesSet,
+			},
+		},
+	})
+}
+
+func devicesList(c *cli.Context) {
+	cfg := getConfig(c)
+	first := true
+	writer := newTableWriter()
+	for _, device := range cfg.Devices {
+		if !first {
+			fmt.Fprintln(writer)
+		}
+		fmt.Fprintln(writer, "ID:\t", device.DeviceID, "\t")
+		fmt.Fprintln(writer, "Name:\t", device.Name, "\t(name)")
+		fmt.Fprintln(writer, "Address:\t", strings.Join(device.Addresses, " "), "\t(address)")
+		fmt.Fprintln(writer, "Compression:\t", device.Compression, "\t(compression)")
+		fmt.Fprintln(writer, "Certificate name:\t", device.CertName, "\t(certname)")
+		fmt.Fprintln(writer, "Introducer:\t", device.Introducer, "\t(introducer)")
+		first = false
+	}
+	writer.Flush()
+}
+
+func devicesAdd(c *cli.Context) {
+	nid := c.Args()[0]
+	id := parseDeviceID(nid)
+
+	newDevice := config.DeviceConfiguration{
+		DeviceID:  id,
+		Name:      nid,
+		Addresses: []string{"dynamic"},
+	}
+
+	if len(c.Args()) > 1 {
+		newDevice.Name = c.Args()[1]
+	}
+
+	if len(c.Args()) > 2 {
+		addresses := c.Args()[2:]
+		for _, item := range addresses {
+			if item == "dynamic" {
+				continue
+			}
+			validAddress(item)
+		}
+		newDevice.Addresses = addresses
+	}
+
+	cfg := getConfig(c)
+	for _, device := range cfg.Devices {
+		if device.DeviceID == id {
+			die("Device " + nid + " already exists")
+		}
+	}
+	cfg.Devices = append(cfg.Devices, newDevice)
+	setConfig(c, cfg)
+}
+
+func devicesRemove(c *cli.Context) {
+	nid := c.Args()[0]
+	id := parseDeviceID(nid)
+	if nid == getMyID(c) {
+		die("Cannot remove yourself")
+	}
+	cfg := getConfig(c)
+	for i, device := range cfg.Devices {
+		if device.DeviceID == id {
+			last := len(cfg.Devices) - 1
+			cfg.Devices[i] = cfg.Devices[last]
+			cfg.Devices = cfg.Devices[:last]
+			setConfig(c, cfg)
+			return
+		}
+	}
+	die("Device " + nid + " not found")
+}
+
+func devicesGet(c *cli.Context) {
+	nid := c.Args()[0]
+	id := parseDeviceID(nid)
+	arg := c.Args()[1]
+	cfg := getConfig(c)
+	for _, device := range cfg.Devices {
+		if device.DeviceID != id {
+			continue
+		}
+		switch strings.ToLower(arg) {
+		case "name":
+			fmt.Println(device.Name)
+		case "address":
+			fmt.Println(strings.Join(device.Addresses, "\n"))
+		case "compression":
+			fmt.Println(device.Compression.String())
+		case "certname":
+			fmt.Println(device.CertName)
+		case "introducer":
+			fmt.Println(device.Introducer)
+		default:
+			die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
+		}
+		return
+	}
+	die("Device " + nid + " not found")
+}
+
+func devicesSet(c *cli.Context) {
+	nid := c.Args()[0]
+	id := parseDeviceID(nid)
+	arg := c.Args()[1]
+	config := getConfig(c)
+	for i, device := range config.Devices {
+		if device.DeviceID != id {
+			continue
+		}
+		switch strings.ToLower(arg) {
+		case "name":
+			config.Devices[i].Name = strings.Join(c.Args()[2:], " ")
+		case "address":
+			for _, item := range c.Args()[2:] {
+				if item == "dynamic" {
+					continue
+				}
+				validAddress(item)
+			}
+			config.Devices[i].Addresses = c.Args()[2:]
+		case "compression":
+			err := config.Devices[i].Compression.UnmarshalText([]byte(c.Args()[2]))
+			die(err)
+		case "certname":
+			config.Devices[i].CertName = strings.Join(c.Args()[2:], " ")
+		case "introducer":
+			config.Devices[i].Introducer = parseBool(c.Args()[2])
+		default:
+			die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
+		}
+		setConfig(c, config)
+		return
+	}
+	die("Device " + nid + " not found")
+}

cmd/stcli/cmd_errors.go

@@ -0,0 +1,67 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/AudriusButkevicius/cli"
+)
+
+func init() {
+	cliCommands = append(cliCommands, cli.Command{
+		Name:     "errors",
+		HideHelp: true,
+		Usage:    "Error command group",
+		Subcommands: []cli.Command{
+			{
+				Name:     "show",
+				Usage:    "Show pending errors",
+				Requires: &cli.Requires{},
+				Action:   errorsShow,
+			},
+			{
+				Name:     "push",
+				Usage:    "Push an error to active clients",
+				Requires: &cli.Requires{"error message..."},
+				Action:   errorsPush,
+			},
+			{
+				Name:     "clear",
+				Usage:    "Clear pending errors",
+				Requires: &cli.Requires{},
+				Action:   wrappedHTTPPost("system/error/clear"),
+			},
+		},
+	})
+}
+
+func errorsShow(c *cli.Context) {
+	response := httpGet(c, "system/error")
+	var data map[string][]map[string]interface{}
+	json.Unmarshal(responseToBArray(response), &data)
+	writer := newTableWriter()
+	for _, item := range data["errors"] {
+		time := item["when"].(string)[:19]
+		time = strings.Replace(time, "T", " ", 1)
+		err := item["message"].(string)
+		err = strings.TrimSpace(err)
+		fmt.Fprintln(writer, time+":\t"+err)
+	}
+	writer.Flush()
+}
+
+func errorsPush(c *cli.Context) {
+	err := strings.Join(c.Args(), " ")
+	response := httpPost(c, "system/error", strings.TrimSpace(err))
+	if response.StatusCode != 200 {
+		err = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
+		body := string(responseToBArray(response))
+		if body != "" {
+			err += "\nBody: " + body
+		}
+		die(err)
+	}
+}

cmd/stcli/cmd_folders.go

@@ -0,0 +1,361 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/AudriusButkevicius/cli"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/fs"
+)
+
+func init() {
+	cliCommands = append(cliCommands, cli.Command{
+		Name:     "folders",
+		HideHelp: true,
+		Usage:    "Folder command group",
+		Subcommands: []cli.Command{
+			{
+				Name:     "list",
+				Usage:    "List available folders",
+				Requires: &cli.Requires{},
+				Action:   foldersList,
+			},
+			{
+				Name:     "add",
+				Usage:    "Add a new folder",
+				Requires: &cli.Requires{"folder id", "directory"},
+				Action:   foldersAdd,
+			},
+			{
+				Name:     "remove",
+				Usage:    "Remove an existing folder",
+				Requires: &cli.Requires{"folder id"},
+				Action:   foldersRemove,
+			},
+			{
+				Name:     "override",
+				Usage:    "Override changes from other nodes for a master folder",
+				Requires: &cli.Requires{"folder id"},
+				Action:   foldersOverride,
+			},
+			{
+				Name:     "get",
+				Usage:    "Get a property of a folder",
+				Requires: &cli.Requires{"folder id", "property"},
+				Action:   foldersGet,
+			},
+			{
+				Name:     "set",
+				Usage:    "Set a property of a folder",
+				Requires: &cli.Requires{"folder id", "property", "value..."},
+				Action:   foldersSet,
+			},
+			{
+				Name:     "unset",
+				Usage:    "Unset a property of a folder",
+				Requires: &cli.Requires{"folder id", "property"},
+				Action:   foldersUnset,
+			},
+			{
+				Name:     "devices",
+				Usage:    "Folder devices command group",
+				HideHelp: true,
+				Subcommands: []cli.Command{
+					{
+						Name:     "list",
+						Usage:    "List of devices which the folder is shared with",
+						Requires: &cli.Requires{"folder id"},
+						Action:   foldersDevicesList,
+					},
+					{
+						Name:     "add",
+						Usage:    "Share a folder with a device",
+						Requires: &cli.Requires{"folder id", "device id"},
+						Action:   foldersDevicesAdd,
+					},
+					{
+						Name:     "remove",
+						Usage:    "Unshare a folder with a device",
+						Requires: &cli.Requires{"folder id", "device id"},
+						Action:   foldersDevicesRemove,
+					},
+					{
+						Name:     "clear",
+						Usage:    "Unshare a folder with all devices",
+						Requires: &cli.Requires{"folder id"},
+						Action:   foldersDevicesClear,
+					},
+				},
+			},
+		},
+	})
+}
+
+func foldersList(c *cli.Context) {
+	cfg := getConfig(c)
+	first := true
+	writer := newTableWriter()
+	for _, folder := range cfg.Folders {
+		if !first {
+			fmt.Fprintln(writer)
+		}
+		fs := folder.Filesystem()
+		fmt.Fprintln(writer, "ID:\t", folder.ID, "\t")
+		fmt.Fprintln(writer, "Path:\t", fs.URI(), "\t(directory)")
+		fmt.Fprintln(writer, "Path type:\t", fs.Type(), "\t(directory-type)")
+		fmt.Fprintln(writer, "Folder type:\t", folder.Type, "\t(type)")
+		fmt.Fprintln(writer, "Ignore permissions:\t", folder.IgnorePerms, "\t(permissions)")
+		fmt.Fprintln(writer, "Rescan interval in seconds:\t", folder.RescanIntervalS, "\t(rescan)")
+
+		if folder.Versioning.Type != "" {
+			fmt.Fprintln(writer, "Versioning:\t", folder.Versioning.Type, "\t(versioning)")
+			for key, value := range folder.Versioning.Params {
+				fmt.Fprintf(writer, "Versioning %s:\t %s \t(versioning-%s)\n", key, value, key)
+			}
+		}
+		first = false
+	}
+	writer.Flush()
+}
+
+func foldersAdd(c *cli.Context) {
+	cfg := getConfig(c)
+	abs, err := filepath.Abs(c.Args()[1])
+	die(err)
+	folder := config.FolderConfiguration{
+		ID:             c.Args()[0],
+		Path:           filepath.Clean(abs),
+		FilesystemType: fs.FilesystemTypeBasic,
+	}
+	cfg.Folders = append(cfg.Folders, folder)
+	setConfig(c, cfg)
+}
+
+func foldersRemove(c *cli.Context) {
+	cfg := getConfig(c)
+	rid := c.Args()[0]
+	for i, folder := range cfg.Folders {
+		if folder.ID == rid {
+			last := len(cfg.Folders) - 1
+			cfg.Folders[i] = cfg.Folders[last]
+			cfg.Folders = cfg.Folders[:last]
+			setConfig(c, cfg)
+			return
+		}
+	}
+	die("Folder " + rid + " not found")
+}
+
+func foldersOverride(c *cli.Context) {
+	cfg := getConfig(c)
+	rid := c.Args()[0]
+	for _, folder := range cfg.Folders {
+		if folder.ID == rid && folder.Type == config.FolderTypeSendOnly {
+			response := httpPost(c, "db/override", "")
+			if response.StatusCode != 200 {
+				err := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
+				body := string(responseToBArray(response))
+				if body != "" {
+					err += "\nBody: " + body
+				}
+				die(err)
+			}
+			return
+		}
+	}
+	die("Folder " + rid + " not found or folder not master")
+}
+
+func foldersGet(c *cli.Context) {
+	cfg := getConfig(c)
+	rid := c.Args()[0]
+	arg := strings.ToLower(c.Args()[1])
+	for _, folder := range cfg.Folders {
+		if folder.ID != rid {
+			continue
+		}
+		if strings.HasPrefix(arg, "versioning-") {
+			arg = arg[11:]
+			value, ok := folder.Versioning.Params[arg]
+			if ok {
+				fmt.Println(value)
+				return
+			}
+			die("Versioning property " + c.Args()[1][11:] + " not found")
+		}
+		switch arg {
+		case "directory":
+			fmt.Println(folder.Filesystem().URI())
+		case "directory-type":
+			fmt.Println(folder.Filesystem().Type())
+		case "type":
+			fmt.Println(folder.Type)
+		case "permissions":
+			fmt.Println(folder.IgnorePerms)
+		case "rescan":
+			fmt.Println(folder.RescanIntervalS)
+		case "versioning":
+			if folder.Versioning.Type != "" {
+				fmt.Println(folder.Versioning.Type)
+			}
+		default:
+			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, directory-type, type, permissions, versioning, versioning-<key>")
+		}
+		return
+	}
+	die("Folder " + rid + " not found")
+}
+
+func foldersSet(c *cli.Context) {
+	rid := c.Args()[0]
+	arg := strings.ToLower(c.Args()[1])
+	val := strings.Join(c.Args()[2:], " ")
+	cfg := getConfig(c)
+	for i, folder := range cfg.Folders {
+		if folder.ID != rid {
+			continue
+		}
+		if strings.HasPrefix(arg, "versioning-") {
+			cfg.Folders[i].Versioning.Params[arg[11:]] = val
+			setConfig(c, cfg)
+			return
+		}
+		switch arg {
+		case "directory":
+			cfg.Folders[i].Path = val
+		case "directory-type":
+			var fsType fs.FilesystemType
+			fsType.UnmarshalText([]byte(val))
+			cfg.Folders[i].FilesystemType = fsType
+		case "type":
+			var t config.FolderType
+			if err := t.UnmarshalText([]byte(val)); err != nil {
+				die("Invalid folder type: " + err.Error())
+			}
+			cfg.Folders[i].Type = t
+		case "permissions":
+			cfg.Folders[i].IgnorePerms = parseBool(val)
+		case "rescan":
+			cfg.Folders[i].RescanIntervalS = parseInt(val)
+		case "versioning":
+			cfg.Folders[i].Versioning.Type = val
+		default:
+			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, master, permissions, versioning, versioning-<key>")
+		}
+		setConfig(c, cfg)
+		return
+	}
+	die("Folder " + rid + " not found")
+}
+
+func foldersUnset(c *cli.Context) {
+	rid := c.Args()[0]
+	arg := strings.ToLower(c.Args()[1])
+	cfg := getConfig(c)
+	for i, folder := range cfg.Folders {
+		if folder.ID != rid {
+			continue
+		}
+		if strings.HasPrefix(arg, "versioning-") {
+			arg = arg[11:]
+			if _, ok := folder.Versioning.Params[arg]; ok {
+				delete(cfg.Folders[i].Versioning.Params, arg)
+				setConfig(c, cfg)
+				return
+			}
+			die("Versioning property " + c.Args()[1][11:] + " not found")
+		}
+		switch arg {
+		case "versioning":
+			cfg.Folders[i].Versioning.Type = ""
+			cfg.Folders[i].Versioning.Params = make(map[string]string)
+		default:
+			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: versioning, versioning-<key>")
+		}
+		setConfig(c, cfg)
+		return
+	}
+	die("Folder " + rid + " not found")
+}
+
+func foldersDevicesList(c *cli.Context) {
+	rid := c.Args()[0]
+	cfg := getConfig(c)
+	for _, folder := range cfg.Folders {
+		if folder.ID != rid {
+			continue
+		}
+		for _, device := range folder.Devices {
+			fmt.Println(device.DeviceID)
+		}
+		return
+	}
+	die("Folder " + rid + " not found")
+}
+
+func foldersDevicesAdd(c *cli.Context) {
+	rid := c.Args()[0]
+	nid := parseDeviceID(c.Args()[1])
+	cfg := getConfig(c)
+	for i, folder := range cfg.Folders {
+		if folder.ID != rid {
+			continue
+		}
+		for _, device := range folder.Devices {
+			if device.DeviceID == nid {
+				die("Device " + c.Args()[1] + " is already part of this folder")
+			}
+		}
+		for _, device := range cfg.Devices {
+			if device.DeviceID == nid {
+				cfg.Folders[i].Devices = append(folder.Devices, config.FolderDeviceConfiguration{
+					DeviceID: device.DeviceID,
+				})
+				setConfig(c, cfg)
+				return
+			}
+		}
+		die("Device " + c.Args()[1] + " not found in device list")
+	}
+	die("Folder " + rid + " not found")
+}
+
+func foldersDevicesRemove(c *cli.Context) {
+	rid := c.Args()[0]
+	nid := parseDeviceID(c.Args()[1])
+	cfg := getConfig(c)
+	for ri, folder := range cfg.Folders {
+		if folder.ID != rid {
+			continue
+		}
+		for ni, device := range folder.Devices {
+			if device.DeviceID == nid {
+				last := len(folder.Devices) - 1
+				cfg.Folders[ri].Devices[ni] = folder.Devices[last]
+				cfg.Folders[ri].Devices = cfg.Folders[ri].Devices[:last]
+				setConfig(c, cfg)
+				return
+			}
+		}
+		die("Device " + c.Args()[1] + " not found")
+	}
+	die("Folder " + rid + " not found")
+}
+
+func foldersDevicesClear(c *cli.Context) {
+	rid := c.Args()[0]
+	cfg := getConfig(c)
+	for i, folder := range cfg.Folders {
+		if folder.ID != rid {
+			continue
+		}
+		cfg.Folders[i].Devices = []config.FolderDeviceConfiguration{}
+		setConfig(c, cfg)
+		return
+	}
+	die("Folder " + rid + " not found")
+}

cmd/stcli/cmd_general.go

@@ -0,0 +1,94 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/AudriusButkevicius/cli"
+)
+
+func init() {
+	cliCommands = append(cliCommands, []cli.Command{
+		{
+			Name:     "id",
+			Usage:    "Get ID of the Syncthing client",
+			Requires: &cli.Requires{},
+			Action:   generalID,
+		},
+		{
+			Name:     "status",
+			Usage:    "Configuration status, whether or not a restart is required for changes to take effect",
+			Requires: &cli.Requires{},
+			Action:   generalStatus,
+		},
+		{
+			Name:     "config",
+			Usage:    "Configuration",
+			Requires: &cli.Requires{},
+			Action:   generalConfiguration,
+		},
+		{
+			Name:     "restart",
+			Usage:    "Restart syncthing",
+			Requires: &cli.Requires{},
+			Action:   wrappedHTTPPost("system/restart"),
+		},
+		{
+			Name:     "shutdown",
+			Usage:    "Shutdown syncthing",
+			Requires: &cli.Requires{},
+			Action:   wrappedHTTPPost("system/shutdown"),
+		},
+		{
+			Name:     "reset",
+			Usage:    "Reset syncthing deleting all folders and devices",
+			Requires: &cli.Requires{},
+			Action:   wrappedHTTPPost("system/reset"),
+		},
+		{
+			Name:     "upgrade",
+			Usage:    "Upgrade syncthing (if a newer version is available)",
+			Requires: &cli.Requires{},
+			Action:   wrappedHTTPPost("system/upgrade"),
+		},
+		{
+			Name:     "version",
+			Usage:    "Syncthing client version",
+			Requires: &cli.Requires{},
+			Action:   generalVersion,
+		},
+	}...)
+}
+
+func generalID(c *cli.Context) {
+	fmt.Println(getMyID(c))
+}
+
+func generalStatus(c *cli.Context) {
+	response := httpGet(c, "system/config/insync")
+	var status struct{ ConfigInSync bool }
+	json.Unmarshal(responseToBArray(response), &status)
+	if !status.ConfigInSync {
+		die("Config out of sync")
+	}
+	fmt.Println("Config in sync")
+}
+
+func generalConfiguration(c *cli.Context) {
+	response := httpGet(c, "system/config")
+	var jsResponse interface{}
+	json.Unmarshal(responseToBArray(response), &jsResponse)
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	enc.Encode(jsResponse)
+}
+
+func generalVersion(c *cli.Context) {
+	response := httpGet(c, "system/version")
+	version := make(map[string]interface{})
+	json.Unmarshal(responseToBArray(response), &version)
+	prettyPrintJSON(version)
+}

cmd/stcli/cmd_gui.go

@@ -0,0 +1,127 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/AudriusButkevicius/cli"
+)
+
+func init() {
+	cliCommands = append(cliCommands, cli.Command{
+		Name:     "gui",
+		HideHelp: true,
+		Usage:    "GUI command group",
+		Subcommands: []cli.Command{
+			{
+				Name:     "dump",
+				Usage:    "Show all GUI configuration settings",
+				Requires: &cli.Requires{},
+				Action:   guiDump,
+			},
+			{
+				Name:     "get",
+				Usage:    "Get a GUI configuration setting",
+				Requires: &cli.Requires{"setting"},
+				Action:   guiGet,
+			},
+			{
+				Name:     "set",
+				Usage:    "Set a GUI configuration setting",
+				Requires: &cli.Requires{"setting", "value"},
+				Action:   guiSet,
+			},
+			{
+				Name:     "unset",
+				Usage:    "Unset a GUI configuration setting",
+				Requires: &cli.Requires{"setting"},
+				Action:   guiUnset,
+			},
+		},
+	})
+}
+
+func guiDump(c *cli.Context) {
+	cfg := getConfig(c).GUI
+	writer := newTableWriter()
+	fmt.Fprintln(writer, "Enabled:\t", cfg.Enabled, "\t(enabled)")
+	fmt.Fprintln(writer, "Use HTTPS:\t", cfg.UseTLS(), "\t(tls)")
+	fmt.Fprintln(writer, "Listen Addresses:\t", cfg.Address(), "\t(address)")
+	if cfg.User != "" {
+		fmt.Fprintln(writer, "Authentication User:\t", cfg.User, "\t(username)")
+		fmt.Fprintln(writer, "Authentication Password:\t", cfg.Password, "\t(password)")
+	}
+	if cfg.APIKey != "" {
+		fmt.Fprintln(writer, "API Key:\t", cfg.APIKey, "\t(apikey)")
+	}
+	writer.Flush()
+}
+
+func guiGet(c *cli.Context) {
+	cfg := getConfig(c).GUI
+	arg := c.Args()[0]
+	switch strings.ToLower(arg) {
+	case "enabled":
+		fmt.Println(cfg.Enabled)
+	case "tls":
+		fmt.Println(cfg.UseTLS())
+	case "address":
+		fmt.Println(cfg.Address())
+	case "user":
+		if cfg.User != "" {
+			fmt.Println(cfg.User)
+		}
+	case "password":
+		if cfg.User != "" {
+			fmt.Println(cfg.Password)
+		}
+	case "apikey":
+		if cfg.APIKey != "" {
+			fmt.Println(cfg.APIKey)
+		}
+	default:
+		die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
+	}
+}
+
+func guiSet(c *cli.Context) {
+	cfg := getConfig(c)
+	arg := c.Args()[0]
+	val := c.Args()[1]
+	switch strings.ToLower(arg) {
+	case "enabled":
+		cfg.GUI.Enabled = parseBool(val)
+	case "tls":
+		cfg.GUI.RawUseTLS = parseBool(val)
+	case "address":
+		validAddress(val)
+		cfg.GUI.RawAddress = val
+	case "user":
+		cfg.GUI.User = val
+	case "password":
+		cfg.GUI.Password = val
+	case "apikey":
+		cfg.GUI.APIKey = val
+	default:
+		die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
+	}
+	setConfig(c, cfg)
+}
+
+func guiUnset(c *cli.Context) {
+	cfg := getConfig(c)
+	arg := c.Args()[0]
+	switch strings.ToLower(arg) {
+	case "user":
+		cfg.GUI.User = ""
+	case "password":
+		cfg.GUI.Password = ""
+	case "apikey":
+		cfg.GUI.APIKey = ""
+	default:
+		die("Invalid setting: " + arg + "\nAvailable settings: user, password, apikey")
+	}
+	setConfig(c, cfg)
+}

cmd/stcli/cmd_options.go

@@ -0,0 +1,173 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/AudriusButkevicius/cli"
+)
+
+func init() {
+	cliCommands = append(cliCommands, cli.Command{
+		Name:     "options",
+		HideHelp: true,
+		Usage:    "Options command group",
+		Subcommands: []cli.Command{
+			{
+				Name:     "dump",
+				Usage:    "Show all Syncthing option settings",
+				Requires: &cli.Requires{},
+				Action:   optionsDump,
+			},
+			{
+				Name:     "get",
+				Usage:    "Get a Syncthing option setting",
+				Requires: &cli.Requires{"setting"},
+				Action:   optionsGet,
+			},
+			{
+				Name:     "set",
+				Usage:    "Set a Syncthing option setting",
+				Requires: &cli.Requires{"setting", "value..."},
+				Action:   optionsSet,
+			},
+		},
+	})
+}
+
+func optionsDump(c *cli.Context) {
+	cfg := getConfig(c).Options
+	writer := newTableWriter()
+
+	fmt.Fprintln(writer, "Sync protocol listen addresses:\t", strings.Join(cfg.ListenAddresses, " "), "\t(addresses)")
+	fmt.Fprintln(writer, "Global discovery enabled:\t", cfg.GlobalAnnEnabled, "\t(globalannenabled)")
+	fmt.Fprintln(writer, "Global discovery servers:\t", strings.Join(cfg.GlobalAnnServers, " "), "\t(globalannserver)")
+
+	fmt.Fprintln(writer, "Local discovery enabled:\t", cfg.LocalAnnEnabled, "\t(localannenabled)")
+	fmt.Fprintln(writer, "Local discovery port:\t", cfg.LocalAnnPort, "\t(localannport)")
+
+	fmt.Fprintln(writer, "Outgoing rate limit in KiB/s:\t", cfg.MaxSendKbps, "\t(maxsend)")
+	fmt.Fprintln(writer, "Incoming rate limit in KiB/s:\t", cfg.MaxRecvKbps, "\t(maxrecv)")
+	fmt.Fprintln(writer, "Reconnect interval in seconds:\t", cfg.ReconnectIntervalS, "\t(reconnect)")
+	fmt.Fprintln(writer, "Start browser:\t", cfg.StartBrowser, "\t(browser)")
+	fmt.Fprintln(writer, "Enable UPnP:\t", cfg.NATEnabled, "\t(nat)")
+	fmt.Fprintln(writer, "UPnP Lease in minutes:\t", cfg.NATLeaseM, "\t(natlease)")
+	fmt.Fprintln(writer, "UPnP Renewal period in minutes:\t", cfg.NATRenewalM, "\t(natrenew)")
+	fmt.Fprintln(writer, "Restart on Wake Up:\t", cfg.RestartOnWakeup, "\t(wake)")
+
+	reporting := "unrecognized value"
+	switch cfg.URAccepted {
+	case -1:
+		reporting = "false"
+	case 0:
+		reporting = "undecided/false"
+	case 1:
+		reporting = "true"
+	}
+	fmt.Fprintln(writer, "Anonymous usage reporting:\t", reporting, "\t(reporting)")
+
+	writer.Flush()
+}
+
+func optionsGet(c *cli.Context) {
+	cfg := getConfig(c).Options
+	arg := c.Args()[0]
+	switch strings.ToLower(arg) {
+	case "address":
+		fmt.Println(strings.Join(cfg.ListenAddresses, "\n"))
+	case "globalannenabled":
+		fmt.Println(cfg.GlobalAnnEnabled)
+	case "globalannservers":
+		fmt.Println(strings.Join(cfg.GlobalAnnServers, "\n"))
+	case "localannenabled":
+		fmt.Println(cfg.LocalAnnEnabled)
+	case "localannport":
+		fmt.Println(cfg.LocalAnnPort)
+	case "maxsend":
+		fmt.Println(cfg.MaxSendKbps)
+	case "maxrecv":
+		fmt.Println(cfg.MaxRecvKbps)
+	case "reconnect":
+		fmt.Println(cfg.ReconnectIntervalS)
+	case "browser":
+		fmt.Println(cfg.StartBrowser)
+	case "nat":
+		fmt.Println(cfg.NATEnabled)
+	case "natlease":
+		fmt.Println(cfg.NATLeaseM)
+	case "natrenew":
+		fmt.Println(cfg.NATRenewalM)
+	case "reporting":
+		switch cfg.URAccepted {
+		case -1:
+			fmt.Println("false")
+		case 0:
+			fmt.Println("undecided/false")
+		case 1:
+			fmt.Println("true")
+		default:
+			fmt.Println("unknown")
+		}
+	case "wake":
+		fmt.Println(cfg.RestartOnWakeup)
+	default:
+		die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
+	}
+}
+
+func optionsSet(c *cli.Context) {
+	config := getConfig(c)
+	arg := c.Args()[0]
+	val := c.Args()[1]
+	switch strings.ToLower(arg) {
+	case "address":
+		for _, item := range c.Args().Tail() {
+			validAddress(item)
+		}
+		config.Options.ListenAddresses = c.Args().Tail()
+	case "globalannenabled":
+		config.Options.GlobalAnnEnabled = parseBool(val)
+	case "globalannserver":
+		for _, item := range c.Args().Tail() {
+			validAddress(item)
+		}
+		config.Options.GlobalAnnServers = c.Args().Tail()
+	case "localannenabled":
+		config.Options.LocalAnnEnabled = parseBool(val)
+	case "localannport":
+		config.Options.LocalAnnPort = parsePort(val)
+	case "maxsend":
+		config.Options.MaxSendKbps = parseUint(val)
+	case "maxrecv":
+		config.Options.MaxRecvKbps = parseUint(val)
+	case "reconnect":
+		config.Options.ReconnectIntervalS = parseUint(val)
+	case "browser":
+		config.Options.StartBrowser = parseBool(val)
+	case "nat":
+		config.Options.NATEnabled = parseBool(val)
+	case "natlease":
+		config.Options.NATLeaseM = parseUint(val)
+	case "natrenew":
+		config.Options.NATRenewalM = parseUint(val)
+	case "reporting":
+		switch strings.ToLower(val) {
+		case "u", "undecided", "unset":
+			config.Options.URAccepted = 0
+		default:
+			boolvalue := parseBool(val)
+			if boolvalue {
+				config.Options.URAccepted = 1
+			} else {
+				config.Options.URAccepted = -1
+			}
+		}
+	case "wake":
+		config.Options.RestartOnWakeup = parseBool(val)
+	default:
+		die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
+	}
+	setConfig(c, config)
+}

cmd/stcli/cmd_report.go

@@ -0,0 +1,72 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/AudriusButkevicius/cli"
+)
+
+func init() {
+	cliCommands = append(cliCommands, cli.Command{
+		Name:     "report",
+		HideHelp: true,
+		Usage:    "Reporting command group",
+		Subcommands: []cli.Command{
+			{
+				Name:     "system",
+				Usage:    "Report system state",
+				Requires: &cli.Requires{},
+				Action:   reportSystem,
+			},
+			{
+				Name:     "connections",
+				Usage:    "Report about connections to other devices",
+				Requires: &cli.Requires{},
+				Action:   reportConnections,
+			},
+			{
+				Name:     "usage",
+				Usage:    "Usage report",
+				Requires: &cli.Requires{},
+				Action:   reportUsage,
+			},
+		},
+	})
+}
+
+func reportSystem(c *cli.Context) {
+	response := httpGet(c, "system/status")
+	data := make(map[string]interface{})
+	json.Unmarshal(responseToBArray(response), &data)
+	prettyPrintJSON(data)
+}
+
+func reportConnections(c *cli.Context) {
+	response := httpGet(c, "system/connections")
+	data := make(map[string]map[string]interface{})
+	json.Unmarshal(responseToBArray(response), &data)
+	var overall map[string]interface{}
+	for key, value := range data {
+		if key == "total" {
+			overall = value
+			continue
+		}
+		value["Device ID"] = key
+		prettyPrintJSON(value)
+		fmt.Println()
+	}
+	if overall != nil {
+		fmt.Println("=== Overall statistics ===")
+		prettyPrintJSON(overall)
+	}
+}
+
+func reportUsage(c *cli.Context) {
+	response := httpGet(c, "svc/report")
+	report := make(map[string]interface{})
+	json.Unmarshal(responseToBArray(response), &report)
+	prettyPrintJSON(report)
+}

cmd/stcli/labels.go

@@ -0,0 +1,31 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+var jsonAttributeLabels = map[string]string{
+	"folderMaxMiB":   "Largest folder size in MiB",
+	"folderMaxFiles": "Largest folder file count",
+	"longVersion":    "Long version",
+	"totMiB":         "Total size in MiB",
+	"totFiles":       "Total files",
+	"uniqueID":       "Unique ID",
+	"numFolders":     "Folder count",
+	"numDevices":     "Device count",
+	"memoryUsageMiB": "Memory usage in MiB",
+	"memorySize":     "Total memory in MiB",
+	"sha256Perf":     "SHA256 Benchmark",
+	"At":             "Last contacted",
+	"Completion":     "Percent complete",
+	"InBytesTotal":   "Total bytes received",
+	"OutBytesTotal":  "Total bytes sent",
+	"ClientVersion":  "Client version",
+	"alloc":          "Memory allocated in bytes",
+	"sys":            "Memory using in bytes",
+	"cpuPercent":     "CPU load in percent",
+	"extAnnounceOK":  "External announcments working",
+	"goroutines":     "Number of Go routines",
+	"myID":           "Client ID",
+	"tilde":          "Tilde expands to",
+	"arch":           "Architecture",
+	"os":             "OS",
+}

cmd/stcli/main.go

@@ -0,0 +1,63 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"sort"
+
+	"github.com/AudriusButkevicius/cli"
+)
+
+type ByAlphabet []cli.Command
+
+func (a ByAlphabet) Len() int           { return len(a) }
+func (a ByAlphabet) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a ByAlphabet) Less(i, j int) bool { return a[i].Name < a[j].Name }
+
+var cliCommands []cli.Command
+
+func main() {
+	app := cli.NewApp()
+	app.Name = "syncthing-cli"
+	app.Author = "Audrius Butkevičius"
+	app.Email = "audrius.butkevicius@gmail.com"
+	app.Usage = "Syncthing command line interface"
+	app.Version = "0.1"
+	app.HideHelp = true
+
+	app.Flags = []cli.Flag{
+		cli.StringFlag{
+			Name:   "endpoint, e",
+			Value:  "http://127.0.0.1:8384",
+			Usage:  "End point to connect to",
+			EnvVar: "STENDPOINT",
+		},
+		cli.StringFlag{
+			Name:   "apikey, k",
+			Value:  "",
+			Usage:  "API Key",
+			EnvVar: "STAPIKEY",
+		},
+		cli.StringFlag{
+			Name:   "username, u",
+			Value:  "",
+			Usage:  "Username",
+			EnvVar: "STUSERNAME",
+		},
+		cli.StringFlag{
+			Name:   "password, p",
+			Value:  "",
+			Usage:  "Password",
+			EnvVar: "STPASSWORD",
+		},
+		cli.BoolFlag{
+			Name:   "insecure, i",
+			Usage:  "Do not verify SSL certificate",
+			EnvVar: "STINSECURE",
+		},
+	}
+
+	sort.Sort(ByAlphabet(cliCommands))
+	app.Commands = cliCommands
+	app.RunAndExitOnError()
+}

cmd/stcli/utils.go

@@ -0,0 +1,165 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+	"unicode"
+
+	"github.com/AudriusButkevicius/cli"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func responseToBArray(response *http.Response) []byte {
+	defer response.Body.Close()
+	bytes, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		die(err)
+	}
+	return bytes
+}
+
+func die(vals ...interface{}) {
+	if len(vals) > 1 || vals[0] != nil {
+		os.Stderr.WriteString(fmt.Sprintln(vals...))
+		os.Exit(1)
+	}
+}
+
+func wrappedHTTPPost(url string) func(c *cli.Context) {
+	return func(c *cli.Context) {
+		httpPost(c, url, "")
+	}
+}
+
+func prettyPrintJSON(json map[string]interface{}) {
+	writer := newTableWriter()
+	remap := make(map[string]interface{})
+	for k, v := range json {
+		key, ok := jsonAttributeLabels[k]
+		if !ok {
+			key = firstUpper(k)
+		}
+		remap[key] = v
+	}
+
+	jsonKeys := make([]string, 0, len(remap))
+	for key := range remap {
+		jsonKeys = append(jsonKeys, key)
+	}
+	sort.Strings(jsonKeys)
+	for _, k := range jsonKeys {
+		var value string
+		rvalue := remap[k]
+		switch rvalue.(type) {
+		case int, int16, int32, int64, uint, uint16, uint32, uint64, float32, float64:
+			value = fmt.Sprintf("%.0f", rvalue)
+		default:
+			value = fmt.Sprint(rvalue)
+		}
+		if value == "" {
+			continue
+		}
+		fmt.Fprintln(writer, k+":\t"+value)
+	}
+	writer.Flush()
+}
+
+func firstUpper(str string) string {
+	for i, v := range str {
+		return string(unicode.ToUpper(v)) + str[i+1:]
+	}
+	return ""
+}
+
+func newTableWriter() *tabwriter.Writer {
+	writer := new(tabwriter.Writer)
+	writer.Init(os.Stdout, 0, 8, 0, '\t', 0)
+	return writer
+}
+
+func getMyID(c *cli.Context) string {
+	response := httpGet(c, "system/status")
+	data := make(map[string]interface{})
+	json.Unmarshal(responseToBArray(response), &data)
+	return data["myID"].(string)
+}
+
+func getConfig(c *cli.Context) config.Configuration {
+	response := httpGet(c, "system/config")
+	config := config.Configuration{}
+	json.Unmarshal(responseToBArray(response), &config)
+	return config
+}
+
+func setConfig(c *cli.Context, cfg config.Configuration) {
+	body, err := json.Marshal(cfg)
+	die(err)
+	response := httpPost(c, "system/config", string(body))
+	if response.StatusCode != 200 {
+		die("Unexpected status code", response.StatusCode)
+	}
+}
+
+func parseBool(input string) bool {
+	val, err := strconv.ParseBool(input)
+	if err != nil {
+		die(input + " is not a valid value for a boolean")
+	}
+	return val
+}
+
+func parseInt(input string) int {
+	val, err := strconv.ParseInt(input, 0, 64)
+	if err != nil {
+		die(input + " is not a valid value for an integer")
+	}
+	return int(val)
+}
+
+func parseUint(input string) int {
+	val, err := strconv.ParseUint(input, 0, 64)
+	if err != nil {
+		die(input + " is not a valid value for an unsigned integer")
+	}
+	return int(val)
+}
+
+func parsePort(input string) int {
+	port := parseUint(input)
+	if port < 1 || port > 65535 {
+		die(input + " is not a valid port\nExpected value between 1 and 65535")
+	}
+	return port
+}
+
+func validAddress(input string) {
+	tokens := strings.Split(input, ":")
+	if len(tokens) != 2 {
+		die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
+	}
+	matched, err := regexp.MatchString("^[a-zA-Z0-9]+([-a-zA-Z0-9.]+[-a-zA-Z0-9]+)?$", tokens[0])
+	die(err)
+	if !matched {
+		die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
+	}
+	parsePort(tokens[1])
+}
+
+func parseDeviceID(input string) protocol.DeviceID {
+	device, err := protocol.DeviceIDFromString(input)
+	if err != nil {
+		die(input + " is not a valid device id")
+	}
+	return device
+}

cmd/stcompdirs/main.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"crypto/md5"
 	"errors"
 	"flag"
 	"fmt"
@@ -14,8 +15,6 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-
-	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 func main() {
@@ -61,7 +60,7 @@ func compareDirectories(dirs ...string) error {
 				} else if res[i].name > res[0].name {
 					return fmt.Errorf("%s missing %v (present in %s)", dirs[i], res[0], dirs[0])
 				}
-				return fmt.Errorf("mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
+				return fmt.Errorf("Mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
 			}
 		}
 
@@ -75,7 +74,7 @@ type fileInfo struct {
 	name string
 	mode os.FileMode
 	mod  int64
-	hash [sha256.Size]byte
+	hash [16]byte
 }
 
 func (f fileInfo) String() string {
@@ -107,7 +106,11 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 			if err != nil {
 				return err
 			}
-			f.hash = sha256.Sum256([]byte(tgt))
+			h := md5.New()
+			h.Write([]byte(tgt))
+			hash := h.Sum(nil)
+
+			copy(f.hash[:], hash)
 		} else if info.IsDir() {
 			f = fileInfo{
 				name: rn,
@@ -120,7 +123,7 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 				mode: info.Mode(),
 				mod:  info.ModTime().Unix(),
 			}
-			sum, err := sha256file(path)
+			sum, err := md5file(path)
 			if err != nil {
 				return err
 			}
@@ -147,14 +150,14 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 	return errc
 }
 
-func sha256file(fname string) (hash [sha256.Size]byte, err error) {
+func md5file(fname string) (hash [16]byte, err error) {
 	f, err := os.Open(fname)
 	if err != nil {
 		return
 	}
 	defer f.Close()
 
-	h := sha256.New()
+	h := md5.New()
 	io.Copy(h, f)
 	hb := h.Sum(nil)
 	copy(hash[:], hb)

cmd/stcrashreceiver/diskstore.go

@@ -1,187 +0,0 @@
-// Copyright (C) 2023 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"io"
-	"log"
-	"os"
-	"path/filepath"
-	"sort"
-	"time"
-)
-
-type diskStore struct {
-	dir      string
-	inbox    chan diskEntry
-	maxBytes int64
-	maxFiles int
-
-	currentFiles []currentFile
-	currentSize  int64
-}
-
-type diskEntry struct {
-	path string
-	data []byte
-}
-
-type currentFile struct {
-	path  string
-	size  int64
-	mtime int64
-}
-
-func (d *diskStore) Serve(ctx context.Context) {
-	if err := os.MkdirAll(d.dir, 0750); err != nil {
-		log.Println("Creating directory:", err)
-		return
-	}
-
-	if err := d.inventory(); err != nil {
-		log.Println("Failed to inventory disk store:", err)
-	}
-	d.clean()
-
-	cleanTimer := time.NewTicker(time.Minute)
-	inventoryTimer := time.NewTicker(24 * time.Hour)
-
-	buf := new(bytes.Buffer)
-	gw := gzip.NewWriter(buf)
-	for {
-		select {
-		case entry := <-d.inbox:
-			path := d.fullPath(entry.path)
-
-			if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
-				log.Println("Creating directory:", err)
-				continue
-			}
-
-			buf.Reset()
-			gw.Reset(buf)
-			if _, err := gw.Write(entry.data); err != nil {
-				log.Println("Failed to compress crash report:", err)
-				continue
-			}
-			if err := gw.Close(); err != nil {
-				log.Println("Failed to compress crash report:", err)
-				continue
-			}
-			if err := os.WriteFile(path, buf.Bytes(), 0644); err != nil {
-				log.Printf("Failed to write %s: %v", entry.path, err)
-				_ = os.Remove(path)
-				continue
-			}
-
-			d.currentSize += int64(buf.Len())
-			d.currentFiles = append(d.currentFiles, currentFile{
-				size: int64(len(entry.data)),
-				path: path,
-			})
-
-		case <-cleanTimer.C:
-			d.clean()
-
-		case <-inventoryTimer.C:
-			if err := d.inventory(); err != nil {
-				log.Println("Failed to inventory disk store:", err)
-			}
-
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-func (d *diskStore) Put(path string, data []byte) bool {
-	select {
-	case d.inbox <- diskEntry{
-		path: path,
-		data: data,
-	}:
-		return true
-	default:
-		return false
-	}
-}
-
-func (d *diskStore) Get(path string) ([]byte, error) {
-	path = d.fullPath(path)
-	bs, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	gr, err := gzip.NewReader(bytes.NewReader(bs))
-	if err != nil {
-		return nil, err
-	}
-	defer gr.Close()
-	return io.ReadAll(gr)
-}
-
-func (d *diskStore) Exists(path string) bool {
-	path = d.fullPath(path)
-	_, err := os.Lstat(path)
-	return err == nil
-}
-
-func (d *diskStore) clean() {
-	for len(d.currentFiles) > 0 && (len(d.currentFiles) > d.maxFiles || d.currentSize > d.maxBytes) {
-		f := d.currentFiles[0]
-		log.Println("Removing", f.path)
-		if err := os.Remove(f.path); err != nil {
-			log.Println("Failed to remove file:", err)
-		}
-		d.currentFiles = d.currentFiles[1:]
-		d.currentSize -= f.size
-	}
-	var oldest time.Duration
-	if len(d.currentFiles) > 0 {
-		oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
-	}
-	log.Printf("Clean complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
-}
-
-func (d *diskStore) inventory() error {
-	d.currentFiles = nil
-	d.currentSize = 0
-	err := filepath.Walk(d.dir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if info.IsDir() {
-			return nil
-		}
-		if filepath.Ext(path) != ".gz" {
-			return nil
-		}
-		d.currentSize += info.Size()
-		d.currentFiles = append(d.currentFiles, currentFile{
-			path:  path,
-			size:  info.Size(),
-			mtime: info.ModTime().Unix(),
-		})
-		return nil
-	})
-	sort.Slice(d.currentFiles, func(i, j int) bool {
-		return d.currentFiles[i].mtime < d.currentFiles[j].mtime
-	})
-	var oldest time.Duration
-	if len(d.currentFiles) > 0 {
-		oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
-	}
-	log.Printf("Inventory complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
-	return err
-}
-
-func (d *diskStore) fullPath(path string) string {
-	return filepath.Join(d.dir, path[0:2], path[2:]) + ".gz"
-}

cmd/stcrashreceiver/main.go

@@ -1,154 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// Command stcrashreceiver is a trivial HTTP server that allows two things:
-//
-// - uploading files (crash reports) named like a SHA256 hash using a PUT request
-// - checking whether such file exists using a HEAD request
-//
-// Typically this should be deployed behind something that manages HTTPS.
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/alecthomas/kong"
-	"github.com/syncthing/syncthing/lib/sha256"
-	"github.com/syncthing/syncthing/lib/ur"
-
-	raven "github.com/getsentry/raven-go"
-)
-
-const maxRequestSize = 1 << 20 // 1 MiB
-
-type cli struct {
-	Dir           string        `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."`
-	DSN           string        `help:"Sentry DSN" env:"SENTRY_DSN"`
-	Listen        string        `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"`
-	MaxDiskFiles  int           `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"`
-	MaxDiskSizeMB int64         `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"`
-	CleanInterval time.Duration `help:"Interval between cleaning up old reports" default:"12h" env:"CLEAN_INTERVAL"`
-	SentryQueue   int           `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"`
-	DiskQueue     int           `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"`
-}
-
-func main() {
-	var params cli
-	kong.Parse(&params)
-
-	mux := http.NewServeMux()
-
-	ds := &diskStore{
-		dir:      filepath.Join(params.Dir, "crash_reports"),
-		inbox:    make(chan diskEntry, params.DiskQueue),
-		maxFiles: params.MaxDiskFiles,
-		maxBytes: params.MaxDiskSizeMB << 20,
-	}
-	go ds.Serve(context.Background())
-
-	ss := &sentryService{
-		dsn:   params.DSN,
-		inbox: make(chan sentryRequest, params.SentryQueue),
-	}
-	go ss.Serve(context.Background())
-
-	cr := &crashReceiver{
-		store:  ds,
-		sentry: ss,
-	}
-
-	mux.Handle("/", cr)
-	mux.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) {
-		w.Write([]byte("OK"))
-	})
-
-	if params.DSN != "" {
-		mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports")))
-	}
-
-	log.SetOutput(os.Stdout)
-	if err := http.ListenAndServe(params.Listen, mux); err != nil {
-		log.Fatalln("HTTP serve:", err)
-	}
-}
-
-func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *http.Request) {
-	return func(w http.ResponseWriter, req *http.Request) {
-		lr := io.LimitReader(req.Body, maxRequestSize)
-		bs, err := io.ReadAll(lr)
-		req.Body.Close()
-		if err != nil {
-			http.Error(w, err.Error(), 500)
-			return
-		}
-
-		var reports []ur.FailureReport
-		err = json.Unmarshal(bs, &reports)
-		if err != nil {
-			http.Error(w, err.Error(), 400)
-			return
-		}
-		if len(reports) == 0 {
-			// Shouldn't happen
-			log.Printf("Got zero failure reports")
-			return
-		}
-
-		version, err := parseVersion(reports[0].Version)
-		if err != nil {
-			http.Error(w, err.Error(), 400)
-			return
-		}
-		for _, r := range reports {
-			pkt := packet(version, "failure")
-			pkt.Message = r.Description
-			pkt.Extra = raven.Extra{
-				"count": r.Count,
-			}
-			for k, v := range r.Extra {
-				pkt.Extra[k] = v
-			}
-			if r.Goroutines != "" {
-				url, err := saveFailureWithGoroutines(r.FailureData, failureDir)
-				if err != nil {
-					log.Println("Saving failure report:", err)
-					http.Error(w, "Internal server error", http.StatusInternalServerError)
-					return
-				}
-				pkt.Extra["goroutinesURL"] = url
-			}
-			message := sanitizeMessageLDB(r.Description)
-			pkt.Fingerprint = []string{message}
-
-			if err := sendReport(dsn, pkt, userIDFor(req)); err != nil {
-				log.Println("Failed to send failure report:", err)
-			} else {
-				log.Println("Sent failure report:", r.Description)
-			}
-		}
-	}
-}
-
-func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string, error) {
-	bs := make([]byte, len(data.Description)+len(data.Goroutines))
-	copy(bs, data.Description)
-	copy(bs[len(data.Description):], data.Goroutines)
-	id := fmt.Sprintf("%x", sha256.Sum256(bs))
-	path := fullPathCompressed(failureDir, id)
-	err := compressAndWrite(bs, path)
-	if err != nil {
-		return "", err
-	}
-	return reportServer + path, nil
-}

cmd/stcrashreceiver/sentry.go

@@ -1,321 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"io"
-	"log"
-	"regexp"
-	"strings"
-	"sync"
-
-	raven "github.com/getsentry/raven-go"
-	"github.com/maruel/panicparse/v2/stack"
-)
-
-const reportServer = "https://crash.syncthing.net/report/"
-
-var loader = newGithubSourceCodeLoader()
-
-func init() {
-	raven.SetSourceCodeLoader(loader)
-}
-
-var (
-	clients    = make(map[string]*raven.Client)
-	clientsMut sync.Mutex
-)
-
-type sentryService struct {
-	dsn   string
-	inbox chan sentryRequest
-}
-
-type sentryRequest struct {
-	reportID string
-	userID   string
-	data     []byte
-}
-
-func (s *sentryService) Serve(ctx context.Context) {
-	for {
-		select {
-		case req := <-s.inbox:
-			pkt, err := parseCrashReport(req.reportID, req.data)
-			if err != nil {
-				log.Println("Failed to parse crash report:", err)
-				continue
-			}
-			if err := sendReport(s.dsn, pkt, req.userID); err != nil {
-				log.Println("Failed to send crash report:", err)
-			}
-
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-func (s *sentryService) Send(reportID, userID string, data []byte) bool {
-	select {
-	case s.inbox <- sentryRequest{reportID, userID, data}:
-		return true
-	default:
-		return false
-	}
-}
-
-func sendReport(dsn string, pkt *raven.Packet, userID string) error {
-	pkt.Interfaces = append(pkt.Interfaces, &raven.User{ID: userID})
-
-	clientsMut.Lock()
-	defer clientsMut.Unlock()
-
-	cli, ok := clients[dsn]
-	if !ok {
-		var err error
-		cli, err = raven.New(dsn)
-		if err != nil {
-			return err
-		}
-		clients[dsn] = cli
-	}
-
-	// The client sets release and such on the packet before sending, in the
-	// misguided idea that it knows this better than than the packet we give
-	// it. So we copy the values from the packet to the client first...
-	cli.SetRelease(pkt.Release)
-	cli.SetEnvironment(pkt.Environment)
-
-	defer cli.Wait()
-	_, errC := cli.Capture(pkt, nil)
-	return <-errC
-}
-
-func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
-	parts := bytes.SplitN(report, []byte("\n"), 2)
-	if len(parts) != 2 {
-		return nil, errors.New("no first line")
-	}
-
-	version, err := parseVersion(string(parts[0]))
-	if err != nil {
-		return nil, err
-	}
-	report = parts[1]
-
-	foundPanic := false
-	var subjectLine []byte
-	for {
-		parts = bytes.SplitN(report, []byte("\n"), 2)
-		if len(parts) != 2 {
-			return nil, errors.New("no panic line found")
-		}
-
-		line := parts[0]
-		report = parts[1]
-
-		if foundPanic {
-			// The previous line was our "Panic at ..." header. We are now
-			// at the beginning of the real panic trace and this is our
-			// subject line.
-			subjectLine = line
-			break
-		} else if bytes.HasPrefix(line, []byte("Panic at")) {
-			foundPanic = true
-		}
-	}
-
-	r := bytes.NewReader(report)
-	ctx, _, err := stack.ScanSnapshot(r, io.Discard, stack.DefaultOpts())
-	if err != nil && err != io.EOF {
-		return nil, err
-	}
-	if ctx == nil || len(ctx.Goroutines) == 0 {
-		return nil, errors.New("no goroutines found")
-	}
-
-	// Lock the source code loader to the version we are processing here.
-	if version.commit != "" {
-		// We have a commit hash, so we know exactly which source to use
-		loader.LockWithVersion(version.commit)
-	} else if strings.HasPrefix(version.tag, "v") {
-		// Lets hope the tag is close enough
-		loader.LockWithVersion(version.tag)
-	} else {
-		// Last resort
-		loader.LockWithVersion("main")
-	}
-	defer loader.Unlock()
-
-	var trace raven.Stacktrace
-	for _, gr := range ctx.Goroutines {
-		if gr.First {
-			trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
-			for i, sc := range gr.Stack.Calls {
-				trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name, sc.RemoteSrcPath, sc.Line, 3, nil)
-			}
-			break
-		}
-	}
-
-	pkt := packet(version, "crash")
-	pkt.Message = string(subjectLine)
-	pkt.Extra = raven.Extra{
-		"url": reportServer + path,
-	}
-	pkt.Interfaces = []raven.Interface{&trace}
-	pkt.Fingerprint = crashReportFingerprint(pkt.Message)
-
-	return pkt, nil
-}
-
-var (
-	indexRe          = regexp.MustCompile(`\[[-:0-9]+\]`)
-	sizeRe           = regexp.MustCompile(`(length|capacity) [0-9]+`)
-	ldbPosRe         = regexp.MustCompile(`(\(pos=)([0-9]+)\)`)
-	ldbChecksumRe    = regexp.MustCompile(`(want=0x)([a-z0-9]+)( got=0x)([a-z0-9]+)`)
-	ldbFileRe        = regexp.MustCompile(`(\[file=)([0-9]+)(\.ldb\])`)
-	ldbInternalKeyRe = regexp.MustCompile(`(internal key ")[^"]+(", len=)[0-9]+`)
-	ldbPathRe        = regexp.MustCompile(`(open|write|read) .+[\\/].+[\\/]index[^\\/]+[\\/][^\\/]+: `)
-)
-
-func sanitizeMessageLDB(message string) string {
-	message = ldbPosRe.ReplaceAllString(message, "${1}x)")
-	message = ldbFileRe.ReplaceAllString(message, "${1}x${3}")
-	message = ldbChecksumRe.ReplaceAllString(message, "${1}X${3}X")
-	message = ldbInternalKeyRe.ReplaceAllString(message, "${1}x${2}x")
-	message = ldbPathRe.ReplaceAllString(message, "$1 x: ")
-	return message
-}
-
-func crashReportFingerprint(message string) []string {
-	// Do not fingerprint on the stack in case of db corruption or fatal
-	// db io error - where it occurs doesn't matter.
-	orig := message
-	message = sanitizeMessageLDB(message)
-	if message != orig {
-		return []string{message}
-	}
-
-	message = indexRe.ReplaceAllString(message, "[x]")
-	message = sizeRe.ReplaceAllString(message, "$1 x")
-
-	// {{ default }} is what sentry uses as a fingerprint by default. While
-	// never specified, the docs point at this being some hash derived from the
-	// stack trace. Here we include the filtered panic message on top of that.
-	// https://docs.sentry.io/platforms/go/data-management/event-grouping/sdk-fingerprinting/#basic-example
-	return []string{"{{ default }}", message}
-}
-
-// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]
-// or, somewhere along the way the "+" in the version tag disappeared:
-// syncthing v1.23.7-dev.26.gdf7b56ae.dirty-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]
-var (
-	longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
-	gitExtraRE    = regexp.MustCompile(`\.\d+\.g[0-9a-f]+`) // ".1.g6aaae618"
-	gitExtraSepRE = regexp.MustCompile(`[.-]`)              // dot or dash
-)
-
-type version struct {
-	version  string   // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
-	tag      string   // "v1.1.4-rc.1"
-	commit   string   // "6aaae618", blank when absent
-	codename string   // "Erbium Earthworm"
-	runtime  string   // "go1.12.5"
-	goos     string   // "darwin"
-	goarch   string   // "amd64"
-	builder  string   // "jb@kvin.kastelo.net"
-	extra    []string // "foo", "bar"
-}
-
-func (v version) environment() string {
-	if v.commit != "" {
-		return "Development"
-	}
-	if strings.Contains(v.tag, "-rc.") {
-		return "Candidate"
-	}
-	if strings.Contains(v.tag, "-") {
-		return "Beta"
-	}
-	return "Stable"
-}
-
-func parseVersion(line string) (version, error) {
-	m := longVersionRE.FindStringSubmatch(line)
-	if len(m) == 0 {
-		return version{}, errors.New("unintelligeble version string")
-	}
-
-	v := version{
-		version:  m[1],
-		codename: m[2],
-		runtime:  m[3],
-		goos:     m[4],
-		goarch:   m[5],
-		builder:  m[6],
-	}
-
-	// Split the version tag into tag and commit. This is old style
-	// v1.2.3-something.4+11-g12345678 or newer with just dots
-	// v1.2.3-something.4.11.g12345678 or v1.2.3-dev.11.g12345678.
-	parts := []string{v.version}
-	if strings.Contains(v.version, "+") {
-		parts = strings.Split(v.version, "+")
-	} else {
-		idxs := gitExtraRE.FindStringIndex(v.version)
-		if len(idxs) > 0 {
-			parts = []string{v.version[:idxs[0]], v.version[idxs[0]+1:]}
-		}
-	}
-	v.tag = parts[0]
-	if len(parts) > 1 {
-		fields := gitExtraSepRE.Split(parts[1], -1)
-		if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
-			v.commit = fields[1][1:]
-		}
-	}
-
-	if len(m) >= 8 && m[7] != "" {
-		tags := strings.Split(m[7], ",")
-		for i := range tags {
-			tags[i] = strings.TrimSpace(tags[i])
-		}
-		v.extra = tags
-	}
-
-	return v, nil
-}
-
-func packet(version version, reportType string) *raven.Packet {
-	pkt := &raven.Packet{
-		Platform:    "go",
-		Release:     version.tag,
-		Environment: version.environment(),
-		Tags: raven.Tags{
-			raven.Tag{Key: "version", Value: version.version},
-			raven.Tag{Key: "tag", Value: version.tag},
-			raven.Tag{Key: "codename", Value: version.codename},
-			raven.Tag{Key: "runtime", Value: version.runtime},
-			raven.Tag{Key: "goos", Value: version.goos},
-			raven.Tag{Key: "goarch", Value: version.goarch},
-			raven.Tag{Key: "builder", Value: version.builder},
-			raven.Tag{Key: "report_type", Value: reportType},
-		},
-	}
-	if version.commit != "" {
-		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
-	}
-	for _, tag := range version.extra {
-		pkt.Tags = append(pkt.Tags, raven.Tag{Key: tag, Value: "1"})
-	}
-	return pkt
-}

cmd/stcrashreceiver/sentry_test.go

@@ -1,155 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"fmt"
-	"os"
-	"testing"
-)
-
-func TestParseVersion(t *testing.T) {
-	cases := []struct {
-		longVersion string
-		parsed      version
-	}{
-		{
-			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`,
-			parsed: version{
-				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
-				tag:      "v1.1.4-rc.1",
-				commit:   "6aaae618",
-				codename: "Erbium Earthworm",
-				runtime:  "go1.12.5",
-				goos:     "darwin",
-				goarch:   "amd64",
-				builder:  "jb@kvin.kastelo.net",
-			},
-		},
-		{
-			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]`,
-			parsed: version{
-				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
-				tag:      "v1.1.4-rc.1",
-				commit:   "6aaae618",
-				codename: "Erbium Earthworm",
-				runtime:  "go1.12.5",
-				goos:     "darwin",
-				goarch:   "amd64",
-				builder:  "jb@kvin.kastelo.net",
-				extra:    []string{"foo", "bar"},
-			},
-		},
-		{
-			longVersion: `syncthing v1.23.7-dev.26.gdf7b56ae-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]`,
-			parsed: version{
-				version:  "v1.23.7-dev.26.gdf7b56ae-stversionextra",
-				tag:      "v1.23.7-dev",
-				commit:   "df7b56ae",
-				codename: "Fermium Flea",
-				runtime:  "go1.20.5",
-				goos:     "darwin",
-				goarch:   "arm64",
-				builder:  "jb@ok.kastelo.net",
-				extra:    []string{"Some Wrapper", "purego", "stnoupgrade"},
-			},
-		},
-	}
-
-	for _, tc := range cases {
-		v, err := parseVersion(tc.longVersion)
-		if err != nil {
-			t.Errorf("%s\nerror: %v\n", tc.longVersion, err)
-			continue
-		}
-		if fmt.Sprint(v) != fmt.Sprint(tc.parsed) {
-			t.Errorf("%s\nA: %v\nE: %v\n", tc.longVersion, v, tc.parsed)
-		}
-	}
-}
-
-func TestParseReport(t *testing.T) {
-	bs, err := os.ReadFile("_testdata/panic.log")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	pkt, err := parseCrashReport("1/2/345", bs)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	bs, err = pkt.JSON()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	fmt.Printf("%s\n", bs)
-}
-
-func TestCrashReportFingerprint(t *testing.T) {
-	cases := []struct {
-		message, exp string
-		ldb          bool
-	}{
-		{
-			message: "panic: leveldb/table: corruption on data-block (pos=51308946): checksum mismatch, want=0xa89f9aa0 got=0xd27cc4c7 [file=004003.ldb]",
-			exp:     "panic: leveldb/table: corruption on data-block (pos=x): checksum mismatch, want=0xX got=0xX [file=x.ldb]",
-			ldb:     true,
-		},
-		{
-			message: "panic: leveldb/table: corruption on table-footer (pos=248): bad magic number [file=001370.ldb]",
-			exp:     "panic: leveldb/table: corruption on table-footer (pos=x): bad magic number [file=x.ldb]",
-			ldb:     true,
-		},
-		{
-			message: "panic: runtime error: slice bounds out of range [4294967283:4194304]",
-			exp:     "panic: runtime error: slice bounds out of range [x]",
-		},
-		{
-			message: "panic: runtime error: slice bounds out of range [-2:]",
-			exp:     "panic: runtime error: slice bounds out of range [x]",
-		},
-		{
-			message: "panic: runtime error: slice bounds out of range [:4294967283] with capacity 32768",
-			exp:     "panic: runtime error: slice bounds out of range [x] with capacity x",
-		},
-		{
-			message: "panic: runtime error: index out of range [0] with length 0",
-			exp:     "panic: runtime error: index out of range [x] with length x",
-		},
-		{
-			message: `panic: leveldb: internal key "\x01", len=1: invalid length`,
-			exp:     `panic: leveldb: internal key "x", len=x: invalid length`,
-			ldb:     true,
-		},
-		{
-			message: `panic: write /var/syncthing/config/index-v0.14.0.db/2732813.log: cannot allocate memory`,
-			exp:     `panic: write x: cannot allocate memory`,
-			ldb:     true,
-		},
-		{
-			message: `panic: filling Blocks: read C:\Users\Serv-Resp-Tizayuca\AppData\Local\Syncthing\index-v0.14.0.db\006561.ldb: Error de datos (comprobación de redundancia cíclica).`,
-			exp:     `panic: filling Blocks: read x: Error de datos (comprobación de redundancia cíclica).`,
-			ldb:     true,
-		},
-	}
-
-	for i, tc := range cases {
-		fingerprint := crashReportFingerprint(tc.message)
-
-		expLen := 2
-		if tc.ldb {
-			expLen = 1
-		}
-		if l := len(fingerprint); l != expLen {
-			t.Errorf("tc %v: Unexpected fingerprint length: %v != %v", i, l, expLen)
-		} else if msg := fingerprint[expLen-1]; msg != tc.exp {
-			t.Errorf("tc %v:\n\"%v\" !=\n\"%v\"", i, msg, tc.exp)
-		}
-	}
-}

cmd/stcrashreceiver/sourcecodeloader.go

@@ -1,118 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"net/http"
-	"path/filepath"
-	"strings"
-	"sync"
-	"time"
-)
-
-const (
-	urlPrefix   = "https://raw.githubusercontent.com/syncthing/syncthing/"
-	httpTimeout = 10 * time.Second
-)
-
-type githubSourceCodeLoader struct {
-	mut     sync.Mutex
-	version string
-	cache   map[string]map[string][][]byte // version -> file -> lines
-	client  *http.Client
-}
-
-func newGithubSourceCodeLoader() *githubSourceCodeLoader {
-	return &githubSourceCodeLoader{
-		cache:  make(map[string]map[string][][]byte),
-		client: &http.Client{Timeout: httpTimeout},
-	}
-}
-
-func (l *githubSourceCodeLoader) LockWithVersion(version string) {
-	l.mut.Lock()
-	l.version = version
-	if _, ok := l.cache[version]; !ok {
-		l.cache[version] = make(map[string][][]byte)
-	}
-}
-
-func (l *githubSourceCodeLoader) Unlock() {
-	l.mut.Unlock()
-}
-
-func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]byte, int) {
-	filename = filepath.ToSlash(filename)
-	lines, ok := l.cache[l.version][filename]
-	if !ok {
-		// Cache whatever we managed to find (or nil if nothing, so we don't try again)
-		defer func() {
-			l.cache[l.version][filename] = lines
-		}()
-
-		knownPrefixes := []string{"/lib/", "/cmd/"}
-		var idx int
-		for _, pref := range knownPrefixes {
-			idx = strings.Index(filename, pref)
-			if idx >= 0 {
-				break
-			}
-		}
-		if idx == -1 {
-			return nil, 0
-		}
-
-		url := urlPrefix + l.version + filename[idx:]
-		resp, err := l.client.Get(url)
-
-		if err != nil {
-			fmt.Println("Loading source:", err)
-			return nil, 0
-		}
-		if resp.StatusCode != http.StatusOK {
-			fmt.Println("Loading source:", resp.Status)
-			return nil, 0
-		}
-		data, err := io.ReadAll(resp.Body)
-		_ = resp.Body.Close()
-		if err != nil {
-			fmt.Println("Loading source:", err.Error())
-			return nil, 0
-		}
-		lines = bytes.Split(data, []byte{'\n'})
-	}
-
-	return getLineFromLines(lines, line, context)
-}
-
-func getLineFromLines(lines [][]byte, line, context int) ([][]byte, int) {
-	if lines == nil {
-		// cached error from ReadFile: return no lines
-		return nil, 0
-	}
-
-	line-- // stack trace lines are 1-indexed
-	start := line - context
-	var idx int
-	if start < 0 {
-		start = 0
-		idx = line
-	} else {
-		idx = context
-	}
-	end := line + context + 1
-	if line >= len(lines) {
-		return nil, 0
-	}
-	if end > len(lines) {
-		end = len(lines)
-	}
-	return lines[start:end], idx
-}

cmd/stcrashreceiver/stcrashreceiver.go

@@ -1,93 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"io"
-	"log"
-	"net/http"
-	"path"
-	"strings"
-)
-
-type crashReceiver struct {
-	store  *diskStore
-	sentry *sentryService
-}
-
-func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	// The final path component should be a SHA256 hash in hex, so 64 hex
-	// characters. We don't care about case on the request but use lower
-	// case internally.
-	reportID := strings.ToLower(path.Base(req.URL.Path))
-	if len(reportID) != 64 {
-		http.Error(w, "Bad request", http.StatusBadRequest)
-		return
-	}
-	for _, c := range reportID {
-		if c >= 'a' && c <= 'f' {
-			continue
-		}
-		if c >= '0' && c <= '9' {
-			continue
-		}
-		http.Error(w, "Bad request", http.StatusBadRequest)
-		return
-	}
-
-	switch req.Method {
-	case http.MethodGet:
-		r.serveGet(reportID, w, req)
-	case http.MethodHead:
-		r.serveHead(reportID, w, req)
-	case http.MethodPut:
-		r.servePut(reportID, w, req)
-	default:
-		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-
-// serveGet responds to GET requests by serving the uncompressed report.
-func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http.Request) {
-	bs, err := r.store.Get(reportID)
-	if err != nil {
-		http.Error(w, "Not found", http.StatusNotFound)
-		return
-	}
-	w.Write(bs)
-}
-
-// serveHead responds to HEAD requests by checking if the named report
-// already exists in the system.
-func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) {
-	if !r.store.Exists(reportID) {
-		http.Error(w, "Not found", http.StatusNotFound)
-	}
-}
-
-// servePut accepts and stores the given report.
-func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *http.Request) {
-	// Read at most maxRequestSize of report data.
-	log.Println("Receiving report", reportID)
-	lr := io.LimitReader(req.Body, maxRequestSize)
-	bs, err := io.ReadAll(lr)
-	if err != nil {
-		log.Println("Reading report:", err)
-		http.Error(w, "Internal server error", http.StatusInternalServerError)
-		return
-	}
-
-	// Store the report
-	if !r.store.Put(reportID, bs) {
-		log.Println("Failed to store report (queue full):", reportID)
-	}
-
-	// Send the report to Sentry
-	if !r.sentry.Send(reportID, userIDFor(req), bs) {
-		log.Println("Failed to send report to sentry (queue full):", reportID)
-	}
-}

cmd/stcrashreceiver/util.go

@@ -1,57 +0,0 @@
-// Copyright (C) 2021 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"net"
-	"net/http"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/sha256"
-)
-
-// userIDFor returns a string we can use as the user ID for the purpose of
-// counting affected users. It's the truncated hash of a salt, the user
-// remote IP, and the current month.
-func userIDFor(req *http.Request) string {
-	addr := req.RemoteAddr
-	if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
-		addr = fwd
-	}
-	if host, _, err := net.SplitHostPort(addr); err == nil {
-		addr = host
-	}
-	now := time.Now().Format("200601")
-	salt := "stcrashreporter"
-	hash := sha256.Sum256([]byte(salt + addr + now))
-	return fmt.Sprintf("%x", hash[:8])
-}
-
-// 01234567890abcdef... => 01/23
-func dirFor(base string) string {
-	return filepath.Join(base[0:2], base[2:4])
-}
-
-func fullPathCompressed(root, reportID string) string {
-	return filepath.Join(root, dirFor(reportID), reportID) + ".gz"
-}
-
-func compressAndWrite(bs []byte, fullPath string) error {
-	// Compress the report for storage
-	buf := new(bytes.Buffer)
-	gw := gzip.NewWriter(buf)
-	_, _ = gw.Write(bs) // can't fail
-	gw.Close()
-
-	// Create an output file with the compressed report
-	return os.WriteFile(fullPath, buf.Bytes(), 0644)
-}

cmd/stdisco/main.go

@@ -7,7 +7,6 @@
 package main
 
 import (
-	"context"
 	"crypto/rand"
 	"encoding/binary"
 	"flag"
@@ -48,16 +47,14 @@ func main() {
 		log.Println("My ID:", myID)
 	}
 
-	ctx := context.Background()
-
-	runbeacon(ctx, beacon.NewMulticast(mc), fake)
-	runbeacon(ctx, beacon.NewBroadcast(bc), fake)
+	runbeacon(beacon.NewMulticast(mc), fake)
+	runbeacon(beacon.NewBroadcast(bc), fake)
 
 	select {}
 }
 
-func runbeacon(ctx context.Context, bc beacon.Interface, fake bool) {
-	go bc.Serve(ctx)
+func runbeacon(bc beacon.Interface, fake bool) {
+	go bc.Serve()
 	go recv(bc)
 	if fake {
 		go send(bc)

cmd/stdiscosrv/apisrv.go

@@ -8,28 +8,22 @@ package main
 
 import (
 	"bytes"
-	"compress/gzip"
 	"context"
 	"crypto/tls"
-	"encoding/base64"
 	"encoding/json"
 	"encoding/pem"
-	"errors"
 	"fmt"
-	io "io"
 	"log"
 	"math/rand"
 	"net"
 	"net/http"
 	"net/url"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/stringutil"
 )
 
 // announcement is the format received from and sent to clients
@@ -71,26 +65,34 @@ func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator,
 	}
 }
 
-func (s *apiSrv) Serve(_ context.Context) error {
+func (s *apiSrv) Serve() {
 	if s.useHTTP {
 		listener, err := net.Listen("tcp", s.addr)
 		if err != nil {
 			log.Println("Listen:", err)
-			return err
+			return
 		}
 		s.listener = listener
 	} else {
 		tlsCfg := &tls.Config{
-			Certificates: []tls.Certificate{s.cert},
-			ClientAuth:   tls.RequestClientCert,
-			MinVersion:   tls.VersionTLS12,
-			NextProtos:   []string{"h2", "http/1.1"},
+			Certificates:           []tls.Certificate{s.cert},
+			ClientAuth:             tls.RequestClientCert,
+			SessionTicketsDisabled: true,
+			MinVersion:             tls.VersionTLS12,
+			CipherSuites: []uint16{
+				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+			},
 		}
 
 		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
 		if err != nil {
 			log.Println("Listen:", err)
-			return err
+			return
 		}
 		s.listener = tlsListener
 	}
@@ -102,16 +104,15 @@ func (s *apiSrv) Serve(_ context.Context) error {
 		ReadTimeout:    httpReadTimeout,
 		WriteTimeout:   httpWriteTimeout,
 		MaxHeaderBytes: httpMaxHeaderBytes,
-		ErrorLog:       log.New(io.Discard, "", 0),
 	}
 
-	err := srv.Serve(s.listener)
-	if err != nil {
+	if err := srv.Serve(s.listener); err != nil {
 		log.Println("Serve:", err)
 	}
-	return err
 }
 
+var topCtx = context.Background()
+
 func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 	t0 := time.Now()
 
@@ -124,25 +125,17 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 	}()
 
 	reqID := requestID(rand.Int63())
-	req = req.WithContext(context.WithValue(req.Context(), idKey, reqID))
+	ctx := context.WithValue(topCtx, idKey, reqID)
 
 	if debug {
-		log.Println(reqID, req.Method, req.URL, req.Proto)
-	}
-
-	remoteAddr := &net.TCPAddr{
-		IP:   nil,
-		Port: -1,
+		log.Println(reqID, req.Method, req.URL)
 	}
 
+	var remoteIP net.IP
 	if s.useHTTP {
-		remoteAddr.IP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
-		if parsedPort, err := strconv.ParseInt(req.Header.Get("X-Client-Port"), 10, 0); err == nil {
-			remoteAddr.Port = int(parsedPort)
-		}
+		remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
 	} else {
-		var err error
-		remoteAddr, err = net.ResolveTCPAddr("tcp", req.RemoteAddr)
+		addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
 		if err != nil {
 			log.Println("remoteAddr:", err)
 			lw.Header().Set("Retry-After", errorRetryAfterString())
@@ -150,20 +143,21 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
 			apiRequestsTotal.WithLabelValues("no_remote_addr").Inc()
 			return
 		}
+		remoteIP = addr.IP
 	}
 
 	switch req.Method {
-	case http.MethodGet:
-		s.handleGET(lw, req)
-	case http.MethodPost:
-		s.handlePOST(remoteAddr, lw, req)
+	case "GET":
+		s.handleGET(ctx, lw, req)
+	case "POST":
+		s.handlePOST(ctx, remoteIP, lw, req)
 	default:
 		http.Error(lw, "Method Not Allowed", http.StatusMethodNotAllowed)
 	}
 }
 
-func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
-	reqID := req.Context().Value(idKey).(requestID)
+func (s *apiSrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
+	reqID := ctx.Value(idKey).(requestID)
 
 	deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
 	if err != nil {
@@ -214,30 +208,21 @@ func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
 
 	lookupRequestsTotal.WithLabelValues("success").Inc()
 
-	w.Header().Set("Content-Type", "application/json")
-	var bw io.Writer = w
-
-	// Use compression if the client asks for it
-	if strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
-		w.Header().Set("Content-Encoding", "gzip")
-		gw := gzip.NewWriter(bw)
-		defer gw.Close()
-		bw = gw
-	}
-
-	json.NewEncoder(bw).Encode(announcement{
-		Seen:      time.Unix(0, rec.Seen).Truncate(time.Second),
+	bs, _ := json.Marshal(announcement{
+		Seen:      time.Unix(0, rec.Seen),
 		Addresses: addressStrs(rec.Addresses),
 	})
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bs)
 }
 
-func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req *http.Request) {
-	reqID := req.Context().Value(idKey).(requestID)
+func (s *apiSrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
+	reqID := ctx.Value(idKey).(requestID)
 
-	rawCert, err := certificateBytes(req)
-	if err != nil {
+	rawCert := certificateBytes(req)
+	if rawCert == nil {
 		if debug {
-			log.Println(reqID, "no certificates:", err)
+			log.Println(reqID, "no certificates")
 		}
 		announceRequestsTotal.WithLabelValues("no_certificate").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
@@ -258,7 +243,7 @@ func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req
 
 	deviceID := protocol.NewDeviceID(rawCert)
 
-	addresses := fixupAddresses(remoteAddr, ann.Addresses)
+	addresses := fixupAddresses(remoteIP, ann.Addresses)
 	if len(addresses) == 0 {
 		announceRequestsTotal.WithLabelValues("bad_request").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
@@ -266,7 +251,7 @@ func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req
 		return
 	}
 
-	if err := s.handleAnnounce(deviceID, addresses); err != nil {
+	if err := s.handleAnnounce(remoteIP, deviceID, addresses); err != nil {
 		announceRequestsTotal.WithLabelValues("internal_error").Inc()
 		w.Header().Set("Retry-After", errorRetryAfterString())
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
@@ -283,7 +268,7 @@ func (s *apiSrv) Stop() {
 	s.listener.Close()
 }
 
-func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string) error {
+func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addresses []string) error {
 	key := deviceID.String()
 	now := time.Now()
 	expire := now.Add(addressExpiryTime).UnixNano()
@@ -294,10 +279,6 @@ func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string)
 		dbAddrs[i].Expires = expire
 	}
 
-	// The address slice must always be sorted for database merges to work
-	// properly.
-	sort.Sort(databaseAddressOrder(dbAddrs))
-
 	seen := now.UnixNano()
 	if s.repl != nil {
 		s.repl.send(key, dbAddrs, seen)
@@ -305,92 +286,42 @@ func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string)
 	return s.db.merge(key, dbAddrs, seen)
 }
 
-func handlePing(w http.ResponseWriter, _ *http.Request) {
+func handlePing(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(204)
 }
 
-func certificateBytes(req *http.Request) ([]byte, error) {
+func certificateBytes(req *http.Request) []byte {
 	if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
-		return req.TLS.PeerCertificates[0].Raw, nil
+		return req.TLS.PeerCertificates[0].Raw
 	}
 
-	var bs []byte
-
 	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
-		if strings.Contains(hdr, "%") {
-			// Nginx using $ssl_client_escaped_cert
-			// The certificate is in PEM format with url encoding.
-			// We need to decode for the PEM decoder
-			hdr, err := url.QueryUnescape(hdr)
-			if err != nil {
-				// Decoding failed
-				return nil, err
-			}
-
-			bs = []byte(hdr)
-		} else {
-			// Nginx using $ssl_client_cert
-			// The certificate is in PEM format but with spaces for newlines. We
-			// need to reinstate the newlines for the PEM decoder. But we need to
-			// leave the spaces in the BEGIN and END lines - the first and last
-			// space - alone.
-			bs = []byte(hdr)
-			firstSpace := bytes.Index(bs, []byte(" "))
-			lastSpace := bytes.LastIndex(bs, []byte(" "))
-			for i := firstSpace + 1; i < lastSpace; i++ {
-				if bs[i] == ' ' {
-					bs[i] = '\n'
-				}
+		bs := []byte(hdr)
+		// The certificate is in PEM format but with spaces for newlines. We
+		// need to reinstate the newlines for the PEM decoder. But we need to
+		// leave the spaces in the BEGIN and END lines - the first and last
+		// space - alone.
+		firstSpace := bytes.Index(bs, []byte(" "))
+		lastSpace := bytes.LastIndex(bs, []byte(" "))
+		for i := firstSpace + 1; i < lastSpace; i++ {
+			if bs[i] == ' ' {
+				bs[i] = '\n'
 			}
 		}
-	} else if hdr := req.Header.Get("X-Tls-Client-Cert-Der-Base64"); hdr != "" {
-		// Caddy {tls_client_certificate_der_base64}
-		hdr, err := base64.StdEncoding.DecodeString(hdr)
-		if err != nil {
+		block, _ := pem.Decode(bs)
+		if block == nil {
 			// Decoding failed
-			return nil, err
+			return nil
 		}
-
-		bs = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: hdr})
-	} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
-		// Traefik 2 passtlsclientcert
-		//
-		// The certificate is in PEM format, maybe with URL encoding
-		// (depends on Traefik version) but without newlines and start/end
-		// statements. We need to decode, reinstate the newlines every 64
-		// character and add statements for the PEM decoder
-
-		if strings.Contains(hdr, "%") {
-			if unesc, err := url.QueryUnescape(hdr); err == nil {
-				hdr = unesc
-			}
-		}
-
-		for i := 64; i < len(hdr); i += 65 {
-			hdr = hdr[:i] + "\n" + hdr[i:]
-		}
-
-		hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
-		hdr += "\n-----END CERTIFICATE-----\n"
-		bs = []byte(hdr)
+		return block.Bytes
 	}
 
-	if bs == nil {
-		return nil, errors.New("empty certificate header")
-	}
-
-	block, _ := pem.Decode(bs)
-	if block == nil {
-		// Decoding failed
-		return nil, errors.New("certificate decode result is empty")
-	}
-
-	return block.Bytes, nil
+	return nil
 }
 
 // fixupAddresses checks the list of addresses, removing invalid ones and
 // replacing unspecified IPs with the given remote IP.
-func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
+func fixupAddresses(remote net.IP, addresses []string) []string {
 	fixed := make([]string, 0, len(addresses))
 	for _, annAddr := range addresses {
 		uri, err := url.Parse(annAddr)
@@ -410,43 +341,33 @@ func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
 			continue
 		}
 
-		if remote != nil {
-			if host == "" || ip.IsUnspecified() {
-				// Replace the unspecified IP with the request source.
+		if host == "" || ip.IsUnspecified() {
+			// Replace the unspecified IP with the request source.
 
-				// ... unless the request source is the loopback address or
-				// multicast/unspecified (can't happen, really).
-				if remote.IP.IsLoopback() || remote.IP.IsMulticast() || remote.IP.IsUnspecified() {
-					continue
-				}
-
-				// Do not use IPv6 remote address if requested scheme is ...4
-				// (i.e., tcp4, etc.)
-				if strings.HasSuffix(uri.Scheme, "4") && remote.IP.To4() == nil {
-					continue
-				}
-
-				// Do not use IPv4 remote address if requested scheme is ...6
-				if strings.HasSuffix(uri.Scheme, "6") && remote.IP.To4() != nil {
-					continue
-				}
-
-				host = remote.IP.String()
+			// ... unless the request source is the loopback address or
+			// multicast/unspecified (can't happen, really).
+			if remote.IsLoopback() || remote.IsMulticast() || remote.IsUnspecified() {
+				continue
 			}
 
-			// If zero port was specified, use remote port.
-			if port == "0" && remote.Port > 0 {
-				port = strconv.Itoa(remote.Port)
+			// Do not use IPv6 remote address if requested scheme is ...4
+			// (i.e., tcp4, etc.)
+			if strings.HasSuffix(uri.Scheme, "4") && remote.To4() == nil {
+				continue
 			}
+
+			// Do not use IPv4 remote address if requested scheme is ...6
+			if strings.HasSuffix(uri.Scheme, "6") && remote.To4() != nil {
+				continue
+			}
+
+			host = remote.String()
 		}
 
 		uri.Host = net.JoinHostPort(host, port)
 		fixed = append(fixed, uri.String())
 	}
 
-	// Remove duplicate addresses
-	fixed = stringutil.UniqueTrimmedStrings(fixed)
-
 	return fixed
 }
 

cmd/stdiscosrv/apisrv_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestFixupAddresses(t *testing.T) {
 	cases := []struct {
-		remote *net.TCPAddr
+		remote net.IP
 		in     []string
 		out    []string
 	}{
@@ -22,53 +22,37 @@ func TestFixupAddresses(t *testing.T) {
 			in:  []string{"tcp://1.2.3.4:22000"},
 			out: []string{"tcp://1.2.3.4:22000"},
 		}, { // unspecified replaced by remote
-			remote: addr("1.2.3.4", 22000),
+			remote: net.ParseIP("1.2.3.4"),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://1.2.3.4:22000", "tcp://192.0.2.42:22000"},
 		}, { // unspecified not used as replacement
-			remote: addr("0.0.0.0", 22000),
+			remote: net.ParseIP("0.0.0.0"),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // unspecified not used as replacement
-			remote: addr("::", 22000),
+			remote: net.ParseIP("::"),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // localhost not used as replacement
-			remote: addr("127.0.0.1", 22000),
+			remote: net.ParseIP("127.0.0.1"),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // localhost not used as replacement
-			remote: addr("::1", 22000),
+			remote: net.ParseIP("::1"),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // multicast not used as replacement
-			remote: addr("224.0.0.1", 22000),
+			remote: net.ParseIP("224.0.0.1"),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // multicast not used as replacement
-			remote: addr("ff80::42", 22000),
+			remote: net.ParseIP("ff80::42"),
 			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
 		}, { // explicitly announced weirdness is also filtered
-			remote: addr("192.0.2.42", 22000),
+			remote: net.ParseIP("192.0.2.42"),
 			in:     []string{"tcp://:22000", "tcp://127.1.2.3:22000", "tcp://[::1]:22000", "tcp://[ff80::42]:22000"},
 			out:    []string{"tcp://192.0.2.42:22000"},
-		}, { // port remapping
-			remote: addr("123.123.123.123", 9000),
-			in:     []string{"tcp://0.0.0.0:0"},
-			out:    []string{"tcp://123.123.123.123:9000"},
-		}, { // unspecified port remapping
-			remote: addr("123.123.123.123", 9000),
-			in:     []string{"tcp://:0"},
-			out:    []string{"tcp://123.123.123.123:9000"},
-		}, { // empty remapping
-			remote: addr("123.123.123.123", 9000),
-			in:     []string{"tcp://"},
-			out:    []string{},
-		}, { // port only remapping
-			remote: addr("123.123.123.123", 9000),
-			in:     []string{"tcp://44.44.44.44:0"},
-			out:    []string{"tcp://44.44.44.44:9000"},
 		},
 	}
 
@@ -79,10 +63,3 @@ func TestFixupAddresses(t *testing.T) {
 		}
 	}
 }
-
-func addr(host string, port int) *net.TCPAddr {
-	return &net.TCPAddr{
-		IP:   net.ParseIP(host),
-		Port: port,
-	}
-}

cmd/stdiscosrv/database.go

@@ -4,20 +4,16 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//go:generate go run ../../proto/scripts/protofmt.go database.proto
+//go:generate go run ../../script/protofmt.go database.proto
 //go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
 
 package main
 
 import (
-	"context"
-	"log"
 	"sort"
 	"time"
 
-	"github.com/syncthing/syncthing/lib/sliceutil"
 	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/storage"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
@@ -40,6 +36,7 @@ type database interface {
 type levelDBStore struct {
 	db         *leveldb.DB
 	inbox      chan func()
+	stop       chan struct{}
 	clock      clock
 	marshalBuf []byte
 }
@@ -52,18 +49,7 @@ func newLevelDBStore(dir string) (*levelDBStore, error) {
 	return &levelDBStore{
 		db:    db,
 		inbox: make(chan func(), 16),
-		clock: defaultClock{},
-	}, nil
-}
-
-func newMemoryLevelDBStore() (*levelDBStore, error) {
-	db, err := leveldb.Open(storage.NewMemStorage(), nil)
-	if err != nil {
-		return nil, err
-	}
-	return &levelDBStore{
-		db:    db,
-		inbox: make(chan func(), 16),
+		stop:  make(chan struct{}),
 		clock: defaultClock{},
 	}, nil
 }
@@ -168,7 +154,7 @@ func (s *levelDBStore) get(key string) (DatabaseRecord, error) {
 	return rec, nil
 }
 
-func (s *levelDBStore) Serve(ctx context.Context) error {
+func (s *levelDBStore) Serve() {
 	t := time.NewTimer(0)
 	defer t.Stop()
 	defer s.db.Close()
@@ -196,7 +182,7 @@ loop:
 			// the next.
 			t.Reset(databaseStatisticsInterval)
 
-		case <-ctx.Done():
+		case <-s.stop:
 			// We're done.
 			close(statisticsTrigger)
 			break loop
@@ -205,8 +191,6 @@ loop:
 
 	// Also wait for statisticsServe to return
 	<-statisticsDone
-
-	return nil
 }
 
 func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
@@ -270,20 +254,21 @@ func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- stru
 	}
 }
 
+func (s *levelDBStore) Stop() {
+	close(s.stop)
+}
+
 // merge returns the merged result of the two database records a and b. The
 // result is the union of the two address sets, with the newer expiry time
 // chosen for any duplicates.
 func merge(a, b DatabaseRecord) DatabaseRecord {
 	// Both lists must be sorted for this to work.
-	if !sort.IsSorted(databaseAddressOrder(a.Addresses)) {
-		log.Println("Warning: bug: addresses not correctly sorted in merge")
-		a.Addresses = sortedAddressCopy(a.Addresses)
-	}
-	if !sort.IsSorted(databaseAddressOrder(b.Addresses)) {
-		// no warning because this is the side we read from disk and it may
-		// legitimately predate correct sorting.
-		b.Addresses = sortedAddressCopy(b.Addresses)
-	}
+	sort.Slice(a.Addresses, func(i, j int) bool {
+		return a.Addresses[i].Address < a.Addresses[j].Address
+	})
+	sort.Slice(b.Addresses, func(i, j int) bool {
+		return b.Addresses[i].Address < b.Addresses[j].Address
+	})
 
 	res := DatabaseRecord{
 		Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
@@ -353,31 +338,17 @@ func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
 	i := 0
 	for i < len(addrs) {
 		if addrs[i].Expires < now {
-			addrs = sliceutil.RemoveAndZero(addrs, i)
+			// This item is expired. Replace it with the last in the list
+			// (noop if we are at the last item).
+			addrs[i] = addrs[len(addrs)-1]
+			// Wipe the last item of the list to release references to
+			// strings and stuff.
+			addrs[len(addrs)-1] = DatabaseAddress{}
+			// Shorten the slice.
+			addrs = addrs[:len(addrs)-1]
 			continue
 		}
 		i++
 	}
 	return addrs
 }
-
-func sortedAddressCopy(addrs []DatabaseAddress) []DatabaseAddress {
-	sorted := make([]DatabaseAddress, len(addrs))
-	copy(sorted, addrs)
-	sort.Sort(databaseAddressOrder(sorted))
-	return sorted
-}
-
-type databaseAddressOrder []DatabaseAddress
-
-func (s databaseAddressOrder) Less(a, b int) bool {
-	return s[a].Address < s[b].Address
-}
-
-func (s databaseAddressOrder) Swap(a, b int) {
-	s[a], s[b] = s[b], s[a]
-}
-
-func (s databaseAddressOrder) Len() int {
-	return len(s)
-}

cmd/stdiscosrv/database.pb.go

@@ -3,14 +3,12 @@
 
 package main
 
-import (
-	fmt "fmt"
-	_ "github.com/gogo/protobuf/gogoproto"
-	proto "github.com/gogo/protobuf/proto"
-	io "io"
-	math "math"
-	math_bits "math/bits"
-)
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import _ "github.com/gogo/protobuf/gogoproto"
+
+import io "io"
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
@@ -21,7 +19,7 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
 
 type DatabaseRecord struct {
 	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
@@ -34,7 +32,7 @@ func (m *DatabaseRecord) Reset()         { *m = DatabaseRecord{} }
 func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
 func (*DatabaseRecord) ProtoMessage()    {}
 func (*DatabaseRecord) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b90fe3356ea5df07, []int{0}
+	return fileDescriptor_database_0f49e029703a04f5, []int{0}
 }
 func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -44,15 +42,15 @@ func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, erro
 		return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
+		n, err := m.MarshalTo(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DatabaseRecord.Merge(m, src)
+func (dst *DatabaseRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseRecord.Merge(dst, src)
 }
 func (m *DatabaseRecord) XXX_Size() int {
 	return m.Size()
@@ -73,7 +71,7 @@ func (m *ReplicationRecord) Reset()         { *m = ReplicationRecord{} }
 func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
 func (*ReplicationRecord) ProtoMessage()    {}
 func (*ReplicationRecord) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b90fe3356ea5df07, []int{1}
+	return fileDescriptor_database_0f49e029703a04f5, []int{1}
 }
 func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -83,15 +81,15 @@ func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, e
 		return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
+		n, err := m.MarshalTo(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ReplicationRecord.Merge(m, src)
+func (dst *ReplicationRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ReplicationRecord.Merge(dst, src)
 }
 func (m *ReplicationRecord) XXX_Size() int {
 	return m.Size()
@@ -111,7 +109,7 @@ func (m *DatabaseAddress) Reset()         { *m = DatabaseAddress{} }
 func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
 func (*DatabaseAddress) ProtoMessage()    {}
 func (*DatabaseAddress) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b90fe3356ea5df07, []int{2}
+	return fileDescriptor_database_0f49e029703a04f5, []int{2}
 }
 func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -121,15 +119,15 @@ func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, err
 		return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
+		n, err := m.MarshalTo(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DatabaseAddress.Merge(m, src)
+func (dst *DatabaseAddress) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseAddress.Merge(dst, src)
 }
 func (m *DatabaseAddress) XXX_Size() int {
 	return m.Size()
@@ -145,34 +143,10 @@ func init() {
 	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
 	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
 }
-
-func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
-
-var fileDescriptor_b90fe3356ea5df07 = []byte{
-	// 270 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
-	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
-	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
-	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
-	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
-	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
-	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
-	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
-	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
-	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
-	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
-	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
-	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
-	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
-	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
-	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
-	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
-}
-
 func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	n, err := m.MarshalTo(dAtA)
 	if err != nil {
 		return nil, err
 	}
@@ -180,51 +154,44 @@ func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
+	var i int
 	_ = i
 	var l int
 	_ = l
-	if m.Missed != 0 {
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
-		i--
-		dAtA[i] = 0x20
-	}
-	if m.Seen != 0 {
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
-		i--
-		dAtA[i] = 0x18
-	}
-	if m.Misses != 0 {
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
-		i--
-		dAtA[i] = 0x10
-	}
 	if len(m.Addresses) > 0 {
-		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
-			{
-				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-				if err != nil {
-					return 0, err
-				}
-				i -= size
-				i = encodeVarintDatabase(dAtA, i, uint64(size))
-			}
-			i--
+		for _, msg := range m.Addresses {
 			dAtA[i] = 0xa
+			i++
+			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
+			n, err := msg.MarshalTo(dAtA[i:])
+			if err != nil {
+				return 0, err
+			}
+			i += n
 		}
 	}
-	return len(dAtA) - i, nil
+	if m.Misses != 0 {
+		dAtA[i] = 0x10
+		i++
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+	}
+	if m.Seen != 0 {
+		dAtA[i] = 0x18
+		i++
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+	}
+	if m.Missed != 0 {
+		dAtA[i] = 0x20
+		i++
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+	}
+	return i, nil
 }
 
 func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	n, err := m.MarshalTo(dAtA)
 	if err != nil {
 		return nil, err
 	}
@@ -232,48 +199,40 @@ func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
+	var i int
 	_ = i
 	var l int
 	_ = l
-	if m.Seen != 0 {
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
-		i--
-		dAtA[i] = 0x18
+	if len(m.Key) > 0 {
+		dAtA[i] = 0xa
+		i++
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		i += copy(dAtA[i:], m.Key)
 	}
 	if len(m.Addresses) > 0 {
-		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
-			{
-				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-				if err != nil {
-					return 0, err
-				}
-				i -= size
-				i = encodeVarintDatabase(dAtA, i, uint64(size))
-			}
-			i--
+		for _, msg := range m.Addresses {
 			dAtA[i] = 0x12
+			i++
+			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
+			n, err := msg.MarshalTo(dAtA[i:])
+			if err != nil {
+				return 0, err
+			}
+			i += n
 		}
 	}
-	if len(m.Key) > 0 {
-		i -= len(m.Key)
-		copy(dAtA[i:], m.Key)
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
-		i--
-		dAtA[i] = 0xa
+	if m.Seen != 0 {
+		dAtA[i] = 0x18
+		i++
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
 	}
-	return len(dAtA) - i, nil
+	return i, nil
 }
 
 func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	n, err := m.MarshalTo(dAtA)
 	if err != nil {
 		return nil, err
 	}
@@ -281,40 +240,32 @@ func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
+	var i int
 	_ = i
 	var l int
 	_ = l
-	if m.Expires != 0 {
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
-		i--
-		dAtA[i] = 0x10
-	}
 	if len(m.Address) > 0 {
-		i -= len(m.Address)
-		copy(dAtA[i:], m.Address)
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
-		i--
 		dAtA[i] = 0xa
+		i++
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
+		i += copy(dAtA[i:], m.Address)
 	}
-	return len(dAtA) - i, nil
+	if m.Expires != 0 {
+		dAtA[i] = 0x10
+		i++
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
+	}
+	return i, nil
 }
 
 func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
-	offset -= sovDatabase(v)
-	base := offset
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
 		v >>= 7
 		offset++
 	}
 	dAtA[offset] = uint8(v)
-	return base
+	return offset + 1
 }
 func (m *DatabaseRecord) Size() (n int) {
 	if m == nil {
@@ -379,7 +330,14 @@ func (m *DatabaseAddress) Size() (n int) {
 }
 
 func sovDatabase(x uint64) (n int) {
-	return (math_bits.Len64(x|1) + 6) / 7
+	for {
+		n++
+		x >>= 7
+		if x == 0 {
+			break
+		}
+	}
+	return n
 }
 func sozDatabase(x uint64) (n int) {
 	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
@@ -399,7 +357,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= uint64(b&0x7F) << shift
+			wire |= (uint64(b) & 0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -427,7 +385,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= int(b&0x7F) << shift
+				msglen |= (int(b) & 0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -436,9 +394,6 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthDatabase
-			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -461,7 +416,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Misses |= int32(b&0x7F) << shift
+				m.Misses |= (int32(b) & 0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -480,7 +435,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= int64(b&0x7F) << shift
+				m.Seen |= (int64(b) & 0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -499,7 +454,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Missed |= int64(b&0x7F) << shift
+				m.Missed |= (int64(b) & 0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -510,7 +465,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if (skippy < 0) || (iNdEx+skippy) < 0 {
+			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
 			if (iNdEx + skippy) > l {
@@ -540,7 +495,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= uint64(b&0x7F) << shift
+			wire |= (uint64(b) & 0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -568,7 +523,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
+				stringLen |= (uint64(b) & 0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -578,9 +533,6 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthDatabase
-			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -600,7 +552,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= int(b&0x7F) << shift
+				msglen |= (int(b) & 0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -609,9 +561,6 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthDatabase
-			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -634,7 +583,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= int64(b&0x7F) << shift
+				m.Seen |= (int64(b) & 0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -645,7 +594,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if (skippy < 0) || (iNdEx+skippy) < 0 {
+			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
 			if (iNdEx + skippy) > l {
@@ -675,7 +624,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= uint64(b&0x7F) << shift
+			wire |= (uint64(b) & 0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -703,7 +652,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
+				stringLen |= (uint64(b) & 0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -713,9 +662,6 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthDatabase
-			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -735,7 +681,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Expires |= int64(b&0x7F) << shift
+				m.Expires |= (int64(b) & 0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -746,7 +692,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if (skippy < 0) || (iNdEx+skippy) < 0 {
+			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
 			if (iNdEx + skippy) > l {
@@ -764,7 +710,6 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 func skipDatabase(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0
-	depth := 0
 	for iNdEx < l {
 		var wire uint64
 		for shift := uint(0); ; shift += 7 {
@@ -796,8 +741,10 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
+			return iNdEx, nil
 		case 1:
 			iNdEx += 8
+			return iNdEx, nil
 		case 2:
 			var length int
 			for shift := uint(0); ; shift += 7 {
@@ -814,34 +761,76 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
+			iNdEx += length
 			if length < 0 {
 				return 0, ErrInvalidLengthDatabase
 			}
-			iNdEx += length
+			return iNdEx, nil
 		case 3:
-			depth++
-		case 4:
-			if depth == 0 {
-				return 0, ErrUnexpectedEndOfGroupDatabase
+			for {
+				var innerWire uint64
+				var start int = iNdEx
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return 0, ErrIntOverflowDatabase
+					}
+					if iNdEx >= l {
+						return 0, io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					innerWire |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				innerWireType := int(innerWire & 0x7)
+				if innerWireType == 4 {
+					break
+				}
+				next, err := skipDatabase(dAtA[start:])
+				if err != nil {
+					return 0, err
+				}
+				iNdEx = start + next
 			}
-			depth--
+			return iNdEx, nil
+		case 4:
+			return iNdEx, nil
 		case 5:
 			iNdEx += 4
+			return iNdEx, nil
 		default:
 			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
 		}
-		if iNdEx < 0 {
-			return 0, ErrInvalidLengthDatabase
-		}
-		if depth == 0 {
-			return iNdEx, nil
-		}
 	}
-	return 0, io.ErrUnexpectedEOF
+	panic("unreachable")
 }
 
 var (
-	ErrInvalidLengthDatabase        = fmt.Errorf("proto: negative length found during unmarshaling")
-	ErrIntOverflowDatabase          = fmt.Errorf("proto: integer overflow")
-	ErrUnexpectedEndOfGroupDatabase = fmt.Errorf("proto: unexpected end of group")
+	ErrInvalidLengthDatabase = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowDatabase   = fmt.Errorf("proto: integer overflow")
 )
+
+func init() { proto.RegisterFile("database.proto", fileDescriptor_database_0f49e029703a04f5) }
+
+var fileDescriptor_database_0f49e029703a04f5 = []byte{
+	// 270 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
+	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
+	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
+	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
+	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
+	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
+	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
+	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
+	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
+	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
+	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
+	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
+	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
+	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
+	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
+	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
+	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
+}

cmd/stdiscosrv/database_test.go

@@ -7,20 +7,21 @@
 package main
 
 import (
-	"context"
 	"fmt"
+	"os"
 	"testing"
 	"time"
 )
 
 func TestDatabaseGetSet(t *testing.T) {
-	db, err := newMemoryLevelDBStore()
+	os.RemoveAll("_database")
+	defer os.RemoveAll("_database")
+	db, err := newLevelDBStore("_database")
 	if err != nil {
 		t.Fatal(err)
 	}
-	ctx, cancel := context.WithCancel(context.Background())
-	go db.Serve(ctx)
-	defer cancel()
+	go db.Serve()
+	defer db.Stop()
 
 	// Check missing record
 
@@ -116,7 +117,7 @@ func TestDatabaseGetSet(t *testing.T) {
 
 	// Put a record with misses
 
-	rec = DatabaseRecord{Misses: 42, Missed: tc.Now().UnixNano()}
+	rec = DatabaseRecord{Misses: 42}
 	if err := db.put("efgh", rec); err != nil {
 		t.Fatal(err)
 	}
@@ -185,7 +186,7 @@ func TestFilter(t *testing.T) {
 		},
 		{
 			a: []DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
-			b: []DatabaseAddress{{Address: "b", Expires: 15}, {Address: "d", Expires: 15}},
+			b: []DatabaseAddress{{Address: "d", Expires: 15}, {Address: "b", Expires: 15}}, // gets reordered
 		},
 	}
 
@@ -206,6 +207,5 @@ func (t *testClock) wind(d time.Duration) {
 }
 
 func (t *testClock) Now() time.Time {
-	t.now = t.now.Add(time.Nanosecond)
 	return t.now
 }

cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv

@@ -1,4 +0,0 @@
-[stdiscosrv]
-title=Syncthing discovery server
-description=Lets syncthing clients discover each other
-ports=8443/tcp

cmd/stdiscosrv/etc/linux-systemd/default

@@ -1,3 +0,0 @@
-# Default settings for syncthing-discosrv (stdiscosrv).
-## Add Options here:
-DISCOSRV_OPTS=

cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service

@@ -1,25 +0,0 @@
-[Unit]
-Description=Syncthing Discovery Server
-After=network.target
-Documentation=man:stdiscosrv(1)
-
-[Service]
-WorkingDirectory=/var/lib/syncthing-discosrv
-EnvironmentFile=/etc/default/syncthing-discosrv
-ExecStart=/usr/bin/stdiscosrv $DISCOSRV_OPTS
-
-# Hardening
-User=syncthing-discosrv
-Group=syncthing
-ProtectSystem=strict
-ReadWritePaths=/var/lib/syncthing-discosrv
-NoNewPrivileges=true
-PrivateTmp=true
-PrivateDevices=true
-ProtectHome=true
-SystemCallArchitectures=native
-MemoryDenyWriteExecute=true
-
-[Install]
-WantedBy=multi-user.target
-Alias=syncthing-discosrv.service

cmd/stdiscosrv/main.go

@@ -7,22 +7,23 @@
 package main
 
 import (
-	"context"
 	"crypto/tls"
 	"flag"
+	"fmt"
 	"log"
 	"net"
 	"net/http"
 	"os"
+	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/thejerf/suture/v4"
+	"github.com/thejerf/suture"
 )
 
 const (
@@ -64,9 +65,34 @@ var levelDBOptions = &opt.Options{
 	WriteBuffer: 32 << 20, // default 4<<20
 }
 
-var debug = false
+var (
+	Version    string
+	BuildStamp string
+	BuildUser  string
+	BuildHost  string
+
+	BuildDate   time.Time
+	LongVersion string
+)
+
+func init() {
+	stamp, _ := strconv.Atoi(BuildStamp)
+	BuildDate = time.Unix(int64(stamp), 0)
+
+	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
+	LongVersion = fmt.Sprintf(`stdiscosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
+}
+
+var (
+	debug = false
+)
 
 func main() {
+	const (
+		cleanIntv = 1 * time.Hour
+		statsIntv = 5 * time.Minute
+	)
+
 	var listen string
 	var dir string
 	var metricsListen string
@@ -74,76 +100,43 @@ func main() {
 	var replicationPeers string
 	var certFile string
 	var keyFile string
-	var replCertFile string
-	var replKeyFile string
 	var useHTTP bool
-	var largeDB bool
 
 	log.SetOutput(os.Stdout)
 	log.SetFlags(0)
 
 	flag.StringVar(&certFile, "cert", "./cert.pem", "Certificate file")
-	flag.StringVar(&keyFile, "key", "./key.pem", "Key file")
 	flag.StringVar(&dir, "db-dir", "./discovery.db", "Database directory")
 	flag.BoolVar(&debug, "debug", false, "Print debug output")
 	flag.BoolVar(&useHTTP, "http", false, "Listen on HTTP (behind an HTTPS proxy)")
 	flag.StringVar(&listen, "listen", ":8443", "Listen address")
+	flag.StringVar(&keyFile, "key", "./key.pem", "Key file")
 	flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
 	flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
 	flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
-	flag.StringVar(&replCertFile, "replication-cert", "", "Certificate file for replication")
-	flag.StringVar(&replKeyFile, "replication-key", "", "Key file for replication")
-	flag.BoolVar(&largeDB, "large-db", false, "Use larger database settings")
-	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
-	log.Println(build.LongVersionFor("stdiscosrv"))
-	if *showVersion {
-		return
-	}
-
-	if largeDB {
-		levelDBOptions.BlockCacheCapacity = 64 << 20
-		levelDBOptions.BlockSize = 64 << 10
-		levelDBOptions.CompactionTableSize = 16 << 20
-		levelDBOptions.CompactionTableSizeMultiplier = 2.0
-		levelDBOptions.WriteBuffer = 64 << 20
-		levelDBOptions.CompactionL0Trigger = 8
-	}
+	log.Println(LongVersion)
 
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
-	if os.IsNotExist(err) {
+	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 20*365)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv")
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
-	} else if err != nil {
-		log.Fatalln("Failed to load keypair:", err)
 	}
+
 	devID := protocol.NewDeviceID(cert.Certificate[0])
 	log.Println("Server device ID is", devID)
 
-	replCert := cert
-	if replCertFile != "" && replKeyFile != "" {
-		replCert, err = tls.LoadX509KeyPair(replCertFile, replKeyFile)
-		if err != nil {
-			log.Fatalln("Failed to load replication keypair:", err)
-		}
-	}
-	replDevID := protocol.NewDeviceID(replCert.Certificate[0])
-	log.Println("Replication device ID is", replDevID)
-
 	// Parse the replication specs, if any.
 	var allowedReplicationPeers []protocol.DeviceID
 	var replicationDestinations []string
 	parts := strings.Split(replicationPeers, ",")
 	for _, part := range parts {
-		if part == "" {
-			continue
-		}
-
 		fields := strings.Split(part, "@")
+
 		switch len(fields) {
 		case 2:
 			// This is an id@address specification. Grab the address for the
@@ -163,9 +156,6 @@ func main() {
 			if err != nil {
 				log.Fatalln("Parsing device ID:", err)
 			}
-			if id == protocol.EmptyDeviceID {
-				log.Fatalf("Missing device ID for peer in %q", part)
-			}
 			allowedReplicationPeers = append(allowedReplicationPeers, id)
 
 		default:
@@ -188,14 +178,14 @@ func main() {
 	// Start any replication senders.
 	var repl replicationMultiplexer
 	for _, dst := range replicationDestinations {
-		rs := newReplicationSender(dst, replCert, allowedReplicationPeers)
+		rs := newReplicationSender(dst, cert, allowedReplicationPeers)
 		main.Add(rs)
 		repl = append(repl, rs)
 	}
 
 	// If we have replication configured, start the replication listener.
 	if len(allowedReplicationPeers) > 0 {
-		rl := newReplicationListener(replicationListen, replCert, allowedReplicationPeers, db)
+		rl := newReplicationListener(replicationListen, cert, allowedReplicationPeers, db)
 		main.Add(rl)
 	}
 
@@ -213,5 +203,5 @@ func main() {
 	}
 
 	// Engage!
-	main.Serve(context.Background())
+	main.Serve()
 }

cmd/stdiscosrv/replication.go

@@ -7,7 +7,6 @@
 package main
 
 import (
-	"context"
 	"crypto/tls"
 	"encoding/binary"
 	"fmt"
@@ -19,11 +18,8 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-const (
-	replicationReadTimeout       = time.Minute
-	replicationWriteTimeout      = 30 * time.Second
-	replicationHeartbeatInterval = time.Second * 30
-)
+const replicationReadTimeout = time.Minute
+const replicationHeartbeatInterval = time.Second * 30
 
 type replicator interface {
 	send(key string, addrs []DatabaseAddress, seen int64)
@@ -36,6 +32,7 @@ type replicationSender struct {
 	cert       tls.Certificate // our certificate
 	allowedIDs []protocol.DeviceID
 	outbox     chan ReplicationRecord
+	stop       chan struct{}
 }
 
 func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
@@ -44,10 +41,11 @@ func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protoco
 		cert:       cert,
 		allowedIDs: allowedIDs,
 		outbox:     make(chan ReplicationRecord, replicationOutboxSize),
+		stop:       make(chan struct{}),
 	}
 }
 
-func (s *replicationSender) Serve(ctx context.Context) error {
+func (s *replicationSender) Serve() {
 	// Sleep a little at startup. Peers often restart at the same time, and
 	// this avoid the service failing and entering backoff state
 	// unnecessarily, while also reducing the reconnect rate to something
@@ -64,30 +62,24 @@ func (s *replicationSender) Serve(ctx context.Context) error {
 	conn, err := tls.Dial("tcp", s.dst, tlsCfg)
 	if err != nil {
 		log.Println("Replication connect:", err)
-		return err
+		return
 	}
 	defer func() {
 		conn.SetWriteDeadline(time.Now().Add(time.Second))
 		conn.Close()
 	}()
 
-	// The replication stream is not especially latency sensitive, but it is
-	// quite a lot of data in small writes. Make it more efficient.
-	if tcpc, ok := conn.NetConn().(*net.TCPConn); ok {
-		_ = tcpc.SetNoDelay(false)
-	}
-
 	// Get the other side device ID.
 	remoteID, err := deviceID(conn)
 	if err != nil {
 		log.Println("Replication connect:", err)
-		return err
+		return
 	}
 
 	// Verify it's in the set of allowed device IDs.
 	if !deviceIDIn(remoteID, s.allowedIDs) {
 		log.Println("Replication connect: unexpected device ID:", remoteID)
-		return err
+		return
 	}
 
 	heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
@@ -125,26 +117,30 @@ func (s *replicationSender) Serve(ctx context.Context) error {
 			binary.BigEndian.PutUint32(buf, uint32(n))
 
 			// Send
-			conn.SetWriteDeadline(time.Now().Add(replicationWriteTimeout))
+			conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
 			if _, err := conn.Write(buf[:4+n]); err != nil {
 				replicationSendsTotal.WithLabelValues("error").Inc()
 				log.Println("Replication write:", err)
-				// Yes, we are losing the replication event here.
-				return err
+				// Yes, we are loosing the replication event here.
+				return
 			}
 			replicationSendsTotal.WithLabelValues("success").Inc()
 
-		case <-ctx.Done():
-			return nil
+		case <-s.stop:
+			return
 		}
 	}
 }
 
+func (s *replicationSender) Stop() {
+	close(s.stop)
+}
+
 func (s *replicationSender) String() string {
 	return fmt.Sprintf("replicationSender(%q)", s.dst)
 }
 
-func (s *replicationSender) send(key string, ps []DatabaseAddress, _ int64) {
+func (s *replicationSender) send(key string, ps []DatabaseAddress, seen int64) {
 	item := ReplicationRecord{
 		Key:       key,
 		Addresses: ps,
@@ -176,6 +172,7 @@ type replicationListener struct {
 	cert       tls.Certificate
 	allowedIDs []protocol.DeviceID
 	db         database
+	stop       chan struct{}
 }
 
 func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
@@ -184,10 +181,11 @@ func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []prot
 		cert:       cert,
 		allowedIDs: allowedIDs,
 		db:         db,
+		stop:       make(chan struct{}),
 	}
 }
 
-func (l *replicationListener) Serve(ctx context.Context) error {
+func (l *replicationListener) Serve() {
 	tlsCfg := &tls.Config{
 		Certificates:       []tls.Certificate{l.cert},
 		ClientAuth:         tls.RequestClientCert,
@@ -198,14 +196,14 @@ func (l *replicationListener) Serve(ctx context.Context) error {
 	lst, err := tls.Listen("tcp", l.addr, tlsCfg)
 	if err != nil {
 		log.Println("Replication listen:", err)
-		return err
+		return
 	}
 	defer lst.Close()
 
 	for {
 		select {
-		case <-ctx.Done():
-			return nil
+		case <-l.stop:
+			return
 		default:
 		}
 
@@ -213,7 +211,7 @@ func (l *replicationListener) Serve(ctx context.Context) error {
 		conn, err := lst.Accept()
 		if err != nil {
 			log.Println("Replication accept:", err)
-			return err
+			return
 		}
 
 		// Figure out the other side device ID
@@ -233,15 +231,19 @@ func (l *replicationListener) Serve(ctx context.Context) error {
 			continue
 		}
 
-		go l.handle(ctx, conn)
+		go l.handle(conn)
 	}
 }
 
+func (l *replicationListener) Stop() {
+	close(l.stop)
+}
+
 func (l *replicationListener) String() string {
 	return fmt.Sprintf("replicationListener(%q)", l.addr)
 }
 
-func (l *replicationListener) handle(ctx context.Context, conn net.Conn) {
+func (l *replicationListener) handle(conn net.Conn) {
 	defer func() {
 		conn.SetWriteDeadline(time.Now().Add(time.Second))
 		conn.Close()
@@ -251,7 +253,7 @@ func (l *replicationListener) handle(ctx context.Context, conn net.Conn) {
 
 	for {
 		select {
-		case <-ctx.Done():
+		case <-l.stop:
 			return
 		default:
 		}

cmd/stdiscosrv/scripts/preinst

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-addgroup --system syncthing
-adduser --system --home /var/lib/syncthing-discosrv --ingroup syncthing syncthing-discosrv

cmd/stdiscosrv/stats.go

@@ -10,7 +10,6 @@ import (
 	"os"
 
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/collectors"
 )
 
 var (
@@ -110,7 +109,7 @@ func init() {
 		databaseKeys, databaseStatisticsSeconds,
 		databaseOperations, databaseOperationSeconds)
 
-	processCollectorOpts := collectors.ProcessCollectorOpts{
+	processCollectorOpts := prometheus.ProcessCollectorOpts{
 		Namespace: "syncthing_discovery",
 		PidFn: func() (int, error) {
 			return os.Getpid(), nil
@@ -118,7 +117,7 @@ func init() {
 	}
 
 	prometheus.MustRegister(
-		collectors.NewProcessCollector(processCollectorOpts),
+		prometheus.NewProcessCollector(processCollectorOpts),
 	)
 
 }

cmd/stevents/main.go

@@ -28,21 +28,16 @@ func main() {
 	log.SetFlags(0)
 
 	target := flag.String("target", "localhost:8384", "Target Syncthing instance")
-	types := flag.String("types", "", "Filter for specific event types (comma-separated)")
 	apikey := flag.String("apikey", "", "Syncthing API key")
 	flag.Parse()
 
 	if *apikey == "" {
 		log.Fatal("Must give -apikey argument")
 	}
-	var eventsArg string
-	if len(*types) > 0 {
-		eventsArg = "&events=" + *types
-	}
 
 	since := 0
 	for {
-		req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d%s", *target, since, eventsArg), nil)
+		req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d", *target, since), nil)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/stfinddevice/main.go

@@ -7,7 +7,6 @@
 package main
 
 import (
-	"context"
 	"crypto/tls"
 	"errors"
 	"flag"
@@ -18,7 +17,6 @@ import (
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
-	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -84,7 +82,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 }
 
 func checkServer(deviceID protocol.DeviceID, server string) checkResult {
-	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger, nil)
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
 	if err != nil {
 		return checkResult{error: err}
 	}
@@ -96,7 +94,7 @@ func checkServer(deviceID protocol.DeviceID, server string) checkResult {
 	})
 
 	go func() {
-		addresses, err := disco.Lookup(context.Background(), deviceID)
+		addresses, err := disco.Lookup(deviceID)
 		res <- checkResult{addresses: addresses, error: err}
 	}()
 

cmd/stfindignored/main.go

@@ -4,7 +4,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// Command stfindignored lists ignored files under a given folder root.
+// Commmand stfindignored lists ignored files under a given folder root.
 package main
 
 import (

cmd/stgenfiles/main.go

@@ -66,7 +66,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
 }
 
 func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
-	src := io.LimitReader(&infiniteReader{fd}, s)
+	src := io.LimitReader(&inifiteReader{fd}, s)
 	dst, err := os.Create(p1)
 	if err != nil {
 		return err
@@ -82,7 +82,7 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
 		return err
 	}
 
-	os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
+	_ = os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
 
 	t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
 	return os.Chtimes(p1, t, t)
@@ -105,11 +105,11 @@ func readRand(bs []byte) (int, error) {
 	return len(bs), nil
 }
 
-type infiniteReader struct {
+type inifiteReader struct {
 	rd io.ReadSeeker
 }
 
-func (i *infiniteReader) Read(bs []byte) (int, error) {
+func (i *inifiteReader) Read(bs []byte) (int, error) {
 	n, err := i.rd.Read(bs)
 	if err == io.EOF {
 		err = nil

cmd/stindex/dump.go

@@ -0,0 +1,83 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/binary"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func dump(ldb *db.Lowlevel) {
+	it := ldb.NewIterator(nil, nil)
+	for it.Next() {
+		key := it.Key()
+		switch key[0] {
+		case db.KeyTypeDevice:
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+			fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
+
+			var f protocol.FileInfo
+			err := f.Unmarshal(it.Value())
+			if err != nil {
+				log.Fatal(err)
+			}
+			fmt.Printf(" V:%v\n", f)
+
+		case db.KeyTypeGlobal:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			var flv db.VersionList
+			flv.Unmarshal(it.Value())
+			fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
+
+		case db.KeyTypeBlock:
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[1+4 : 1+4+32]
+			name := nulString(key[1+4+32:])
+			fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
+
+		case db.KeyTypeDeviceStatistic:
+			fmt.Printf("[dstat] K:%x V:%x\n", it.Key(), it.Value())
+
+		case db.KeyTypeFolderStatistic:
+			fmt.Printf("[fstat] K:%x V:%x\n", it.Key(), it.Value())
+
+		case db.KeyTypeVirtualMtime:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			val := it.Value()
+			var real, virt time.Time
+			real.UnmarshalBinary(val[:len(val)/2])
+			virt.UnmarshalBinary(val[len(val)/2:])
+			fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, real, virt)
+
+		case db.KeyTypeFolderIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
+
+		case db.KeyTypeDeviceIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			val := it.Value()
+			if len(val) == 0 {
+				fmt.Printf("[deviceidx] K:%d V:<nil>\n", key)
+			} else {
+				dev := protocol.DeviceIDFromBytes(val)
+				fmt.Printf("[deviceidx] K:%d V:%s\n", key, dev)
+			}
+
+		default:
+			fmt.Printf("[???]\n  %x\n  %x\n", it.Key(), it.Value())
+		}
+	}
+}

cmd/stindex/dumpsize.go

@@ -4,38 +4,46 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package cli
+package main
 
 import (
+	"container/heap"
 	"encoding/binary"
 	"fmt"
-	"sort"
-
-	"github.com/urfave/cli"
 
 	"github.com/syncthing/syncthing/lib/db"
 )
 
-func indexDumpSize(*cli.Context) error {
-	type sizedElement struct {
-		key  string
-		size int
-	}
+type SizedElement struct {
+	key  string
+	size int
+}
 
-	ldb, err := getDB()
-	if err != nil {
-		return err
-	}
+type ElementHeap []SizedElement
 
-	it, err := ldb.NewPrefixIterator(nil)
-	if err != nil {
-		return err
-	}
+func (h ElementHeap) Len() int           { return len(h) }
+func (h ElementHeap) Less(i, j int) bool { return h[i].size > h[j].size }
+func (h ElementHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
 
-	var elems []sizedElement
+func (h *ElementHeap) Push(x interface{}) {
+	*h = append(*h, x.(SizedElement))
+}
+
+func (h *ElementHeap) Pop() interface{} {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	*h = old[0 : n-1]
+	return x
+}
+
+func dumpsize(ldb *db.Lowlevel) {
+	h := &ElementHeap{}
+	heap.Init(h)
+
+	it := ldb.NewIterator(nil, nil)
+	var ele SizedElement
 	for it.Next() {
-		var ele sizedElement
-
 		key := it.Key()
 		switch key[0] {
 		case db.KeyTypeDevice:
@@ -76,15 +84,11 @@ func indexDumpSize(*cli.Context) error {
 			ele.key = fmt.Sprintf("UNKNOWN:%x", key)
 		}
 		ele.size = len(it.Value())
-		elems = append(elems, ele)
+		heap.Push(h, ele)
 	}
 
-	sort.Slice(elems, func(i, j int) bool {
-		return elems[i].size > elems[j].size
-	})
-	for _, ele := range elems {
+	for h.Len() > 0 {
+		ele = heap.Pop(h).(SizedElement)
 		fmt.Println(ele.key, ele.size)
 	}
-
-	return nil
 }

cmd/stindex/idxck.go

@@ -4,16 +4,12 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at https://mozilla.org/MPL/2.0/.
 
-package cli
+package main
 
 import (
 	"bytes"
 	"encoding/binary"
-	"errors"
 	"fmt"
-	"sort"
-
-	"github.com/urfave/cli"
 
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/protocol"
@@ -35,12 +31,7 @@ type sequenceKey struct {
 	sequence uint64
 }
 
-func indexCheck(*cli.Context) (err error) {
-	ldb, err := getDB()
-	if err != nil {
-		return err
-	}
-
+func idxck(ldb *db.Lowlevel) (success bool) {
 	folders := make(map[uint32]string)
 	devices := make(map[uint32]string)
 	deviceToIDs := make(map[string]uint32)
@@ -48,26 +39,10 @@ func indexCheck(*cli.Context) (err error) {
 	globals := make(map[globalKey]db.VersionList)
 	sequences := make(map[sequenceKey]string)
 	needs := make(map[globalKey]struct{})
-	blocklists := make(map[string]struct{})
-	versions := make(map[string]protocol.Vector)
-	usedBlocklists := make(map[string]struct{})
-	usedVersions := make(map[string]struct{})
 	var localDeviceKey uint32
-	success := true
-	defer func() {
-		if err == nil {
-			if success {
-				fmt.Println("Index check completed successfully.")
-			} else {
-				err = errors.New("Inconsistencies found in the index")
-			}
-		}
-	}()
+	success = true
 
-	it, err := ldb.NewPrefixIterator(nil)
-	if err != nil {
-		return err
-	}
+	it := ldb.NewIterator(nil, nil)
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {
@@ -119,20 +94,6 @@ func indexCheck(*cli.Context) (err error) {
 			folder := binary.BigEndian.Uint32(key[1:])
 			name := nulString(key[1+4:])
 			needs[globalKey{folder, name}] = struct{}{}
-
-		case db.KeyTypeBlockList:
-			hash := string(key[1:])
-			blocklists[hash] = struct{}{}
-
-		case db.KeyTypeVersion:
-			hash := string(key[1:])
-			var v protocol.Vector
-			if err := v.Unmarshal(it.Value()); err != nil {
-				fmt.Println("Unable to unmarshal Vector:", err)
-				success = false
-				continue
-			}
-			versions[hash] = v
 		}
 	}
 
@@ -142,7 +103,6 @@ func indexCheck(*cli.Context) (err error) {
 		return
 	}
 
-	var missingSeq []sequenceKey
 	for fk, fi := range fileInfos {
 		if fk.name != fi.Name {
 			fmt.Printf("Mismatching FileInfo name, %q (key) != %q (actual)\n", fk.name, fi.Name)
@@ -161,11 +121,9 @@ func indexCheck(*cli.Context) (err error) {
 		}
 
 		if fk.device == localDeviceKey {
-			sk := sequenceKey{fk.folder, uint64(fi.Sequence)}
-			name, ok := sequences[sk]
+			name, ok := sequences[sequenceKey{fk.folder, uint64(fi.Sequence)}]
 			if !ok {
 				fmt.Printf("Sequence entry missing for FileInfo %q, folder %q, seq %d\n", fi.Name, folder, fi.Sequence)
-				missingSeq = append(missingSeq, sk)
 				success = false
 				continue
 			}
@@ -174,58 +132,6 @@ func indexCheck(*cli.Context) (err error) {
 				success = false
 			}
 		}
-
-		if len(fi.Blocks) == 0 && len(fi.BlocksHash) != 0 {
-			key := string(fi.BlocksHash)
-			if _, ok := blocklists[key]; !ok {
-				fmt.Printf("Missing block list for file %q, block list hash %x\n", fi.Name, fi.BlocksHash)
-				success = false
-			} else {
-				usedBlocklists[key] = struct{}{}
-			}
-		}
-
-		if fi.VersionHash != nil {
-			key := string(fi.VersionHash)
-			if _, ok := versions[key]; !ok {
-				fmt.Printf("Missing version vector for file %q, version hash %x\n", fi.Name, fi.VersionHash)
-				success = false
-			} else {
-				usedVersions[key] = struct{}{}
-			}
-		}
-
-		_, ok := globals[globalKey{fk.folder, fk.name}]
-		if !ok {
-			fmt.Printf("Missing global for file %q\n", fi.Name)
-			success = false
-			continue
-		}
-	}
-
-	// Aggregate the ranges of missing sequence entries, print them
-
-	sort.Slice(missingSeq, func(a, b int) bool {
-		if missingSeq[a].folder != missingSeq[b].folder {
-			return missingSeq[a].folder < missingSeq[b].folder
-		}
-		return missingSeq[a].sequence < missingSeq[b].sequence
-	})
-
-	var folder uint32
-	var startSeq, prevSeq uint64
-	for _, sk := range missingSeq {
-		if folder != sk.folder || sk.sequence != prevSeq+1 {
-			if folder != 0 {
-				fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
-			}
-			startSeq = sk.sequence
-			folder = sk.folder
-		}
-		prevSeq = sk.sequence
-	}
-	if folder != 0 {
-		fmt.Printf("Folder %d missing %d sequence entries: #%d - #%d\n", folder, prevSeq-startSeq+1, startSeq, prevSeq)
 	}
 
 	for gk, vl := range globals {
@@ -234,10 +140,10 @@ func indexCheck(*cli.Context) (err error) {
 			fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
 			success = false
 		}
-		checkGlobal := func(i int, device []byte, version protocol.Vector, invalid, deleted bool) {
-			dev, ok := deviceToIDs[string(device)]
+		for i, fv := range vl.Versions {
+			dev, ok := deviceToIDs[string(fv.Device)]
 			if !ok {
-				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, device)
+				fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, fv.Device)
 				success = false
 			}
 			fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
@@ -245,31 +151,14 @@ func indexCheck(*cli.Context) (err error) {
 				fmt.Printf("VersionList %q, folder %q, entry %d refers to unknown FileInfo\n", gk.name, folder, i)
 				success = false
 			}
-
-			fiv := fi.Version
-			if fi.VersionHash != nil {
-				fiv = versions[string(fi.VersionHash)]
-			}
-			if !fiv.Equal(version) {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, version, fi.Version)
+			if !fi.Version.Equal(fv.Version) {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Version, fi.Version)
 				success = false
 			}
-			if fi.IsInvalid() != invalid {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, invalid, fi.IsInvalid())
+			if fi.IsInvalid() != fv.Invalid {
+				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Invalid, fi.IsInvalid())
 				success = false
 			}
-			if fi.IsDeleted() != deleted {
-				fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo deleted mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, deleted, fi.IsDeleted())
-				success = false
-			}
-		}
-		for i, fv := range vl.RawVersions {
-			for _, device := range fv.Devices {
-				checkGlobal(i, device, fv.Version, false, fv.Deleted)
-			}
-			for _, device := range fv.InvalidDevices {
-				checkGlobal(i, device, fv.Version, true, fv.Deleted)
-			}
 		}
 
 		// If we need this file we should have a need entry for it. False
@@ -278,10 +167,8 @@ func indexCheck(*cli.Context) (err error) {
 		if needsLocally(vl) {
 			_, ok := needs[gk]
 			if !ok {
-				fv, _ := vl.GetGlobal()
-				devB, _ := fv.FirstDevice()
-				dev := deviceToIDs[string(devB)]
-				fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
+				dev, _ := deviceToIDs[string(vl.Versions[0].Device)]
+				fi, _ := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
 				if !fi.IsDeleted() && !fi.IsIgnored() {
 					fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
 				}
@@ -337,21 +224,19 @@ func indexCheck(*cli.Context) (err error) {
 		}
 	}
 
-	if d := len(blocklists) - len(usedBlocklists); d > 0 {
-		fmt.Printf("%d block list entries out of %d needs GC\n", d, len(blocklists))
-	}
-	if d := len(versions) - len(usedVersions); d > 0 {
-		fmt.Printf("%d version entries out of %d needs GC\n", d, len(versions))
-	}
-
-	return nil
+	return
 }
 
 func needsLocally(vl db.VersionList) bool {
-	gfv, gok := vl.GetGlobal()
-	if !gok { // That's weird, but we hardly need something non-existent
-		return false
+	var lv *protocol.Vector
+	for _, fv := range vl.Versions {
+		if bytes.Equal(fv.Device, protocol.LocalDeviceID[:]) {
+			lv = &fv.Version
+			break
+		}
 	}
-	fv, ok := vl.Get(protocol.LocalDeviceID[:])
-	return db.Need(gfv, ok, fv.Version)
+	if lv == nil {
+		return true // proviosinally, it looks like we need the file
+	}
+	return !lv.GreaterEqual(vl.Versions[0].Version)
 }

cmd/stindex/main.go

@@ -0,0 +1,49 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/syncthing/syncthing/lib/db"
+)
+
+func main() {
+	var mode string
+	log.SetFlags(0)
+	log.SetOutput(os.Stdout)
+
+	flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize, idxck")
+
+	flag.Parse()
+
+	path := flag.Arg(0)
+	if path == "" {
+		path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
+	}
+
+	ldb, err := db.OpenRO(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if mode == "dump" {
+		dump(ldb)
+	} else if mode == "dumpsize" {
+		dumpsize(ldb)
+	} else if mode == "idxck" {
+		if !idxck(ldb) {
+			os.Exit(1)
+		}
+	} else {
+		fmt.Println("Unknown mode")
+	}
+}

cmd/stindex/util.go

@@ -0,0 +1,52 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/syncthing/syncthing/lib/fs"
+)
+
+func nulString(bs []byte) string {
+	for i := range bs {
+		if bs[i] == 0 {
+			return string(bs[:i])
+		}
+	}
+	return string(bs)
+}
+
+func defaultConfigDir() string {
+	switch runtime.GOOS {
+	case "windows":
+		if p := os.Getenv("LocalAppData"); p != "" {
+			return filepath.Join(p, "Syncthing")
+		}
+		return filepath.Join(os.Getenv("AppData"), "Syncthing")
+
+	case "darwin":
+		dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
+		if err != nil {
+			log.Fatal(err)
+		}
+		return dir
+
+	default:
+		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
+			return filepath.Join(xdgCfg, "syncthing")
+		}
+		dir, err := fs.ExpandTilde("~/.config/syncthing")
+		if err != nil {
+			log.Fatal(err)
+		}
+		return dir
+	}
+}

cmd/strelaypoolsrv/auto/noassets.go

@@ -1,16 +0,0 @@
-// Copyright (C) 2021 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//go:build noassets
-// +build noassets
-
-package auto
-
-import "github.com/syncthing/syncthing/lib/assets"
-
-func Assets() map[string]assets.Asset {
-	return nil
-}

cmd/strelaypoolsrv/gui/index.html

@@ -9,15 +9,8 @@
     <meta name="author" content=""/>
 
     <title>Relay stats</title>
-    <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"/>
-    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
-    <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"/>
-    <link rel="stylesheet" href="https://unpkg.com/leaflet@1.6.0/dist/leaflet.css"
-  integrity="sha512-xwE/Az9zrjBIphAcBb3F6JVqxf46+CDLwfLMHloNu6KEQCAWi6HcDUbeOfBIptF7tcCzusKFjFw2yuvEpDL9wQ=="
-  crossorigin=""/>
-  <script src="https://unpkg.com/leaflet@1.6.0/dist/leaflet.js"
-  integrity="sha512-gZwIG9x3wUXg2hdXF6+rVkLF/0Vi9U8D2Ntg4Ga5I5BZpVkVxlJWbSQtXPSiUTtC0TjtGOmxa1AJPuV0CPthew=="
-  crossorigin=""></script>
+    <link href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
+    <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.0.13/css/all.css"/>
 
     <style>
       #map {
@@ -45,19 +38,17 @@
     <div class="container">
       <h1>Relay Pool Data</h1>
       <div ng-if="relays === undefined" class="text-center">
-        <img src="https://cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
-        <p>Please wait while we gather data…</p>
+        <img src="//cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
+        <p>Please wait while we gather data</p>
       </div>
       <div>
         <div ng-show="relays !== undefined" class="ng-hide">
           <p>
-            The relays listed on this page are not managed or vetted by the Syncthing project.
-            Each relay is the responsibility of the relay operator.
-            Currently {{ relays.length }} relays are online.
+            Currently {{ relays.length }} relays online ({{ totals.goMaxProcs }} cores in total).
           </p>
         </div>
         <div id="map"></div> <!-- Can't hide the map, otherwise it freaks out -->
-        <p>The circle size represents how much bytes the relay has transferred relatively to other relays.</p>
+        <p>The circle size represents how much bytes the relay transferred relative to other relays</p>
       </div>
       <div>
         <table class="table table-striped table-condensed table">
@@ -188,22 +179,23 @@
       <hr>
       <p>
         This product includes GeoLite2 data created by MaxMind, available from
-        <a href="https://www.maxmind.com">https://www.maxmind.com</a>.
+        <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
       </p>
     </div>
 
 
-    <script type="text/javascript" src="https://code.jquery.com/jquery-2.1.4.min.js"></script>
-    <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
-    <script type="text/javascript" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
+    <script type="text/javascript" src="//code.jquery.com/jquery-2.1.4.min.js"></script>
+    <script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
+    <script type="text/javascript" src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
+    <script type="text/javascript" src="//maps.googleapis.com/maps/api/js?key=AIzaSyDk5WJ8s7ueLKb99X5DbQ-vkWtPDAKqYs0"></script>
   </body>
 
   <script>
   angular.module('syncthing', [
   ])
-  .config(['$httpProvider', function($httpProvider) {
+  .config(function($httpProvider) {
     $httpProvider.defaults.timeout = 5000;
-  }])
+  })
   .filter('bytes', function() {
     return function(bytes, precision) {
       if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) return '-';
@@ -236,12 +228,11 @@
       numProxies: 0,
       uptimeSeconds: 0,
     };
-    $scope.map = L.map('map').setView([40.90296, 1.90925], 2);
-    L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png',
-    {
-      attribution: 'Leaflet',
-      maxZoom: 17
-    }).addTo($scope.map);
+    $scope.map = new google.maps.Map(document.getElementById('map'), {
+      zoom: 1,
+      mapTypeId: google.maps.MapTypeId.ROADMAP
+    });
+    $scope.mapBounds = new google.maps.LatLngBounds();
     $scope.tooltipTemplate = $('#infoTemplate').html();
     $scope.usedLocations = {};
     $scope.sortType = 'stats.numActiveSessions';
@@ -288,9 +279,8 @@
         }
       });
 
+      $scope.map.fitBounds($scope.mapBounds);
       if ($scope.relays.length == 1) {
-        //Center to only relay with zoom
-        $scope.map.panTo(new L.LatLng(relays[0].location.latitude, relays[0].location.longitude));
         $scope.map.setZoom(13);
       }
     });
@@ -310,50 +300,44 @@
 
         var locParts = loc.split(',');
 
-        relay.marker = new L.Marker([relay.location.latitude, relay.location.longitude],{
+        relay.marker = new google.maps.Marker({
+          map: $scope.map,
+          position: new google.maps.LatLng(locParts[0], locParts[1]),
           title: relay.url,
         });
 
         var scope = $rootScope.$new(true);
         scope.relay = relay;
 
-        var icon = new L.Icon({
-          iconSize:     [18, 28], // size of the icon
-          iconAnchor:   [9, 28], // point of the icon which will correspond to marker's location
-          shadowAnchor: [0, 0],  // the same for the shadow
-          popupAnchor:  [0, -27], // popup anchor
-          shadowSize:   [0,0],
-          iconUrl: 'https://cdn.rawgit.com/pointhi/leaflet-color-markers/master/img/marker-icon-red.png',
-          shadowUrl: 'https://cdnjs.cloudflare.com/ajax/libs/leaflet/0.7.7/images/marker-shadow.png',
+        relay.marker.info = new google.maps.InfoWindow({
+          content: $compile($scope.tooltipTemplate)(scope)[0],
         });
 
-        relay.marker = new L.marker(new L.latLng(locParts[0], locParts[1]),{icon})
-                            .bindPopup($compile($scope.tooltipTemplate)(scope)[0],{})
-                            .on('mouseover', function (e) {
-                                this.openPopup();
-                            }).on('mouseout', function (e) {
-                                this.closePopup();
-                            }).addTo($scope.map);
-
         relay.showMarker = function() {
-          relay.marker.openPopup();
+          relay.marker.info.open($scope.map, relay.marker);
         }
-        
-        relay.hideMarker = function() {
-          relay.marker.closePopup();
-        }
-      }
 
+        relay.hideMarker = function() {
+          relay.marker.info.close();
+        }
+
+        relay.marker.addListener('mouseover', relay.showMarker);
+        relay.marker.addListener('mouseout', relay.hideMarker);
+
+        $scope.mapBounds.extend(relay.marker.position);
+    }
 
     function addCircleToMap(relay) {
-      console.log(relay.location.latitude)
-      L.circle([relay.location.latitude, relay.location.longitude], 
-        {
-          radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000,
-          color: "FF0000",
-          fillColor: "#FF0000",
-          fillOpacity: 0.35,
-        }).addTo($scope.map);
+      relay.marker.circle = new google.maps.Circle({
+        strokeColor: '#FF0000',
+        strokeOpacity: 0.8,
+        strokeWeight: 2,
+        fillColor: '#FF0000',
+        fillOpacity: 0.35,
+        map: $scope.map,
+        center: relay.marker.position,
+        radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000
+      });
     }
 
     function constructURI(url) {

cmd/strelaypoolsrv/main.go

@@ -1,15 +1,20 @@
 // Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
 
+//go:generate go run ../../script/genassets.go gui >auto/gui.go
+
 package main
 
 import (
-	"context"
+	"bytes"
+	"compress/gzip"
 	"crypto/tls"
-	"crypto/x509"
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"log"
+	"math/rand"
+	"mime"
 	"net"
 	"net/http"
 	"net/url"
@@ -17,22 +22,17 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"sync/atomic"
 	"time"
 
-	lru "github.com/hashicorp/golang-lru/v2"
-	"github.com/syncthing/syncthing/lib/httpcache"
-	"github.com/syncthing/syncthing/lib/protocol"
-
+	"github.com/golang/groupcache/lru"
 	"github.com/oschwald/geoip2-golang"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
-	"github.com/syncthing/syncthing/lib/assets"
-	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/relay/client"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/tlsutil"
+	"golang.org/x/time/rate"
 )
 
 type location struct {
@@ -92,27 +92,38 @@ type result struct {
 }
 
 var (
-	testCert          tls.Certificate
-	knownRelaysFile   = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
-	listen            = ":80"
-	dir               string
-	evictionTime      = time.Hour
-	debug             bool
-	permRelaysFile    string
-	ipHeader          string
-	geoipPath         string
-	proto             string
-	statsRefresh      = time.Minute
-	requestQueueLen   = 64
-	requestProcessors = 8
+	testCert        tls.Certificate
+	knownRelaysFile = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
+	listen          = ":80"
+	dir             string
+	evictionTime    = time.Hour
+	debug           bool
+	getLRUSize      = 10 << 10
+	getLimitBurst   = 10
+	getLimitAvg     = 2
+	postLRUSize     = 1 << 10
+	postLimitBurst  = 2
+	postLimitAvg    = 2
+	getLimit        time.Duration
+	postLimit       time.Duration
+	permRelaysFile  string
+	ipHeader        string
+	geoipPath       string
+	proto           string
+	statsRefresh    = time.Minute / 2
 
-	requests chan request
+	getMut      = sync.NewRWMutex()
+	getLRUCache *lru.Cache
+
+	postMut      = sync.NewRWMutex()
+	postLRUCache *lru.Cache
+
+	requests = make(chan request, 10)
 
 	mut             = sync.NewRWMutex()
 	knownRelays     = make([]*relay, 0)
 	permanentRelays = make([]*relay, 0)
 	evictionTimers  = make(map[string]*time.Timer)
-	globalBlocklist = newErrorTracker(1000)
 )
 
 const (
@@ -120,25 +131,29 @@ const (
 )
 
 func main() {
-	log.SetOutput(os.Stdout)
-	log.SetFlags(log.Lshortfile)
-
 	flag.StringVar(&listen, "listen", listen, "Listen address")
 	flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
 	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
 	flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
+	flag.IntVar(&getLRUSize, "get-limit-cache", getLRUSize, "Get request limiter cache size")
+	flag.IntVar(&getLimitAvg, "get-limit-avg", getLimitAvg, "Allowed average get request rate, per 10 s")
+	flag.IntVar(&getLimitBurst, "get-limit-burst", getLimitBurst, "Allowed burst get requests")
+	flag.IntVar(&postLRUSize, "post-limit-cache", postLRUSize, "Post request limiter cache size")
+	flag.IntVar(&postLimitAvg, "post-limit-avg", postLimitAvg, "Allowed average post request rate, per minute")
+	flag.IntVar(&postLimitBurst, "post-limit-burst", postLimitBurst, "Allowed burst post requests")
 	flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
-	flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays")
 	flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
 	flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
 	flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
 	flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
-	flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
-	flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
 
 	flag.Parse()
 
-	requests = make(chan request, requestQueueLen)
+	getLimit = 10 * time.Second / time.Duration(getLimitAvg)
+	postLimit = time.Minute / time.Duration(postLimitAvg)
+
+	getLRUCache = lru.New(getLRUSize)
+	postLRUCache = lru.New(postLRUSize)
 
 	var listener net.Listener
 	var err error
@@ -149,9 +164,7 @@ func main() {
 
 	testCert = createTestCertificate()
 
-	for i := 0; i < requestProcessors; i++ {
-		go requestProcessor()
-	}
+	go requestProcessor()
 
 	// Load relays from cache in the background.
 	// Load them in a serial fashion to make sure any genuine requests
@@ -185,7 +198,6 @@ func main() {
 		tlsCfg := &tls.Config{
 			Certificates: []tls.Certificate{cert},
 			MinVersion:   tls.VersionTLS10, // No SSLv3
-			ClientAuth:   tls.RequestClientCert,
 			CipherSuites: []uint16{
 				// No RC4
 				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
@@ -215,7 +227,7 @@ func main() {
 
 	handler := http.NewServeMux()
 	handler.HandleFunc("/", handleAssets)
-	handler.Handle("/endpoint", httpcache.SinglePath(http.HandlerFunc(handleRequest), 15*time.Second))
+	handler.HandleFunc("/endpoint", handleRequest)
 	handler.HandleFunc("/metrics", handleMetrics)
 
 	srv := http.Server{
@@ -241,42 +253,107 @@ func handleMetrics(w http.ResponseWriter, r *http.Request) {
 func handleAssets(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Cache-Control", "no-cache, must-revalidate")
 
+	assets := auto.Assets()
 	path := r.URL.Path[1:]
 	if path == "" {
 		path = "index.html"
 	}
 
-	as, ok := auto.Assets()[path]
+	bs, ok := assets[path]
 	if !ok {
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
 
-	assets.Serve(w, r, as)
+	etag := fmt.Sprintf("%d", auto.Generated)
+	modified := time.Unix(auto.Generated, 0).UTC()
+
+	w.Header().Set("Last-Modified", modified.Format(http.TimeFormat))
+	w.Header().Set("Etag", etag)
+
+	mtype := mimeTypeForFile(path)
+	if len(mtype) != 0 {
+		w.Header().Set("Content-Type", mtype)
+	}
+
+	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modified.Add(time.Second).After(t) {
+		w.WriteHeader(http.StatusNotModified)
+		return
+	}
+
+	if match := r.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+		w.Header().Set("Content-Encoding", "gzip")
+	} else {
+		// ungzip if browser not send gzip accepted header
+		var gr *gzip.Reader
+		gr, _ = gzip.NewReader(bytes.NewReader(bs))
+		bs, _ = ioutil.ReadAll(gr)
+		gr.Close()
+	}
+	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs)))
+
+	w.Write(bs)
+}
+
+func mimeTypeForFile(file string) string {
+	// We use a built in table of the common types since the system
+	// TypeByExtension might be unreliable. But if we don't know, we delegate
+	// to the system.
+	ext := filepath.Ext(file)
+	switch ext {
+	case ".htm", ".html":
+		return "text/html"
+	case ".css":
+		return "text/css"
+	case ".js":
+		return "application/javascript"
+	case ".json":
+		return "application/json"
+	case ".png":
+		return "image/png"
+	case ".ttf":
+		return "application/x-font-ttf"
+	case ".woff":
+		return "application/x-font-woff"
+	case ".svg":
+		return "image/svg+xml"
+	default:
+		return mime.TypeByExtension(ext)
+	}
 }
 
 func handleRequest(w http.ResponseWriter, r *http.Request) {
 	timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
 
-	w = NewLoggingResponseWriter(w)
+	lw := NewLoggingResponseWriter(w)
+
 	defer func() {
 		timer.ObserveDuration()
-		lw := w.(*loggingResponseWriter)
 		apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
 	}()
 
 	if ipHeader != "" {
-		hdr := r.Header.Get(ipHeader)
-		fields := strings.Split(hdr, ",")
-		if len(fields) > 0 {
-			r.RemoteAddr = strings.TrimSpace(fields[len(fields)-1])
-		}
+		r.RemoteAddr = r.Header.Get(ipHeader)
 	}
 	w.Header().Set("Access-Control-Allow-Origin", "*")
 	switch r.Method {
 	case "GET":
+		if limit(r.RemoteAddr, getLRUCache, getMut, getLimit, getLimitBurst) {
+			w.WriteHeader(httpStatusEnhanceYourCalm)
+			return
+		}
 		handleGetRequest(w, r)
 	case "POST":
+		if limit(r.RemoteAddr, postLRUCache, postMut, postLimit, postLimitBurst) {
+			w.WriteHeader(httpStatusEnhanceYourCalm)
+			return
+		}
 		handlePostRequest(w, r)
 	default:
 		if debug {
@@ -286,46 +363,24 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func handleGetRequest(rw http.ResponseWriter, r *http.Request) {
-	rw.Header().Set("Content-Type", "application/json; charset=utf-8")
-
+func handleGetRequest(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	mut.RLock()
-	relays := make([]*relay, len(permanentRelays)+len(knownRelays))
-	n := copy(relays, permanentRelays)
-	copy(relays[n:], knownRelays)
+	relays := append(permanentRelays, knownRelays...)
 	mut.RUnlock()
 
 	// Shuffle
-	rand.Shuffle(relays)
+	for i := range relays {
+		j := rand.Intn(i + 1)
+		relays[i], relays[j] = relays[j], relays[i]
+	}
 
-	_ = json.NewEncoder(rw).Encode(map[string][]*relay{
+	json.NewEncoder(w).Encode(map[string][]*relay{
 		"relays": relays,
 	})
 }
 
 func handlePostRequest(w http.ResponseWriter, r *http.Request) {
-	// Get the IP address of the client
-	rhost := r.RemoteAddr
-	if host, _, err := net.SplitHostPort(rhost); err == nil {
-		rhost = host
-	}
-
-	// Check the black list. A client is blacklisted if their last 10
-	// attempts to join have all failed. The "Unauthorized" status return
-	// causes strelaysrv to cease attempting to join.
-	if globalBlocklist.IsBlocked(rhost) {
-		log.Println("Rejected blocked client", rhost)
-		http.Error(w, "Too many errors", http.StatusUnauthorized)
-		globalBlocklist.ClearErrors(rhost)
-		return
-	}
-
-	var relayCert *x509.Certificate
-	if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
-		relayCert = r.TLS.PeerCertificates[0]
-		log.Printf("Got TLS cert from relay server")
-	}
-
 	var newRelay relay
 	err := json.NewDecoder(r.Body).Decode(&newRelay)
 	r.Body.Close()
@@ -334,7 +389,7 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		if debug {
 			log.Println("Failed to parse payload")
 		}
-		http.Error(w, err.Error(), http.StatusBadRequest)
+		http.Error(w, err.Error(), 500)
 		return
 	}
 
@@ -343,40 +398,31 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		if debug {
 			log.Println("Failed to parse URI", newRelay.URL)
 		}
-		http.Error(w, err.Error(), http.StatusBadRequest)
+		http.Error(w, err.Error(), 500)
 		return
 	}
 
-	// Canonicalize the URL. In particular, parse and re-encode the query
-	// string so that it's guaranteed to be valid.
-	uri.RawQuery = uri.Query().Encode()
-	newRelay.URL = uri.String()
-
-	if relayCert != nil {
-		advertisedId := uri.Query().Get("id")
-		idFromCert := protocol.NewDeviceID(relayCert.Raw).String()
-		if advertisedId != idFromCert {
-			log.Println("Warning: Relay server requested to join with an ID different from the join request, rejecting")
-			http.Error(w, "mismatched advertised id and join request cert", http.StatusBadRequest)
-			return
-		}
-	}
-
 	host, port, err := net.SplitHostPort(uri.Host)
 	if err != nil {
 		if debug {
 			log.Println("Failed to split URI", newRelay.URL)
 		}
-		http.Error(w, err.Error(), http.StatusBadRequest)
+		http.Error(w, err.Error(), 500)
 		return
 	}
 
+	// Get the IP address of the client
+	rhost := r.RemoteAddr
+	if host, _, err := net.SplitHostPort(rhost); err == nil {
+		rhost = host
+	}
+
 	ip := net.ParseIP(host)
 	// The client did not provide an IP address, use the IP address of the client.
 	if ip == nil || ip.IsUnspecified() {
 		uri.Host = net.JoinHostPort(rhost, port)
 		newRelay.URL = uri.String()
-	} else if host != rhost && relayCert == nil {
+	} else if host != rhost {
 		if debug {
 			log.Println("IP address advertised does not match client IP address", r.RemoteAddr, uri)
 		}
@@ -402,14 +448,10 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 	case requests <- request{&newRelay, reschan, prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("queue"))}:
 		result := <-reschan
 		if result.err != nil {
-			log.Println("Join from", r.RemoteAddr, "failed:", result.err)
-			globalBlocklist.AddError(rhost)
 			relayTestsTotal.WithLabelValues("failed").Inc()
 			http.Error(w, result.err.Error(), http.StatusBadRequest)
 			return
 		}
-		log.Println("Join from", r.RemoteAddr, "succeeded")
-		globalBlocklist.ClearErrors(rhost)
 		relayTestsTotal.WithLabelValues("success").Inc()
 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
 		json.NewEncoder(w).Encode(map[string]time.Duration{
@@ -441,11 +483,11 @@ func handleRelayTest(request request) {
 	if debug {
 		log.Println("Request for", request.relay)
 	}
-	if err := client.TestRelay(context.TODO(), request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3); err != nil {
+	if !client.TestRelay(request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
 		if debug {
-			log.Println("Test for relay", request.relay, "failed:", err)
+			log.Println("Test for relay", request.relay, "failed")
 		}
-		request.result <- result{err, 0}
+		request.result <- result{fmt.Errorf("connection test failed"), 0}
 		return
 	}
 
@@ -457,7 +499,7 @@ func handleRelayTest(request request) {
 		updateMetrics(request.relay.uri.Host, *stats, location)
 	}
 	request.relay.Stats = stats
-	request.relay.StatsRetrieved = time.Now().Truncate(time.Second)
+	request.relay.StatsRetrieved = time.Now()
 	request.relay.Location = location
 
 	timer, ok := evictionTimers[request.relay.uri.Host]
@@ -523,8 +565,30 @@ func evict(relay *relay) func() {
 	}
 }
 
+func limit(addr string, cache *lru.Cache, lock sync.RWMutex, intv time.Duration, burst int) bool {
+	if host, _, err := net.SplitHostPort(addr); err == nil {
+		addr = host
+	}
+
+	lock.RLock()
+	bkt, ok := cache.Get(addr)
+	lock.RUnlock()
+	if ok {
+		bkt := bkt.(*rate.Limiter)
+		if !bkt.Allow() {
+			// Rate limit
+			return true
+		}
+	} else {
+		lock.Lock()
+		cache.Add(addr, rate.NewLimiter(rate.Every(intv), burst))
+		lock.Unlock()
+	}
+	return false
+}
+
 func loadRelays(file string) []*relay {
-	content, err := os.ReadFile(file)
+	content, err := ioutil.ReadFile(file)
 	if err != nil {
 		log.Println("Failed to load relays: " + err.Error())
 		return nil
@@ -532,7 +596,7 @@ func loadRelays(file string) []*relay {
 
 	var relays []*relay
 	for _, line := range strings.Split(string(content), "\n") {
-		if line == "" {
+		if len(line) == 0 {
 			continue
 		}
 
@@ -562,17 +626,17 @@ func saveRelays(file string, relays []*relay) error {
 	for _, relay := range relays {
 		content += relay.uri.String() + "\n"
 	}
-	return os.WriteFile(file, []byte(content), 0o777)
+	return ioutil.WriteFile(file, []byte(content), 0777)
 }
 
 func createTestCertificate() tls.Certificate {
-	tmpDir, err := os.MkdirTemp("", "relaypoolsrv")
+	tmpDir, err := ioutil.TempDir("", "relaypoolsrv")
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
-	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 20*365)
+	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv")
 	if err != nil {
 		log.Fatalln("Failed to create test X509 key pair:", err)
 	}
@@ -621,42 +685,3 @@ func (lrw *loggingResponseWriter) WriteHeader(code int) {
 	lrw.statusCode = code
 	lrw.ResponseWriter.WriteHeader(code)
 }
-
-type errorTracker struct {
-	errors *lru.TwoQueueCache[string, *errorCounter]
-}
-
-type errorCounter struct {
-	count atomic.Int32
-}
-
-func newErrorTracker(size int) *errorTracker {
-	cache, err := lru.New2Q[string, *errorCounter](size)
-	if err != nil {
-		panic(err)
-	}
-	return &errorTracker{
-		errors: cache,
-	}
-}
-
-func (b *errorTracker) AddError(host string) {
-	entry, ok := b.errors.Get(host)
-	if !ok {
-		entry = &errorCounter{}
-		b.errors.Add(host, entry)
-	}
-	c := entry.count.Add(1)
-	log.Printf("Error count for %s is now %d", host, c)
-}
-
-func (b *errorTracker) ClearErrors(host string) {
-	b.errors.Remove(host)
-}
-
-func (b *errorTracker) IsBlocked(host string) bool {
-	if be, ok := b.errors.Get(host); ok {
-		return be.count.Load() > 10
-	}
-	return false
-}

cmd/strelaypoolsrv/main_test.go

@@ -1,94 +0,0 @@
-// Copyright © 2020 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"net/http/httptest"
-	"net/url"
-	"strings"
-	"sync"
-	"testing"
-)
-
-func init() {
-	for i := 0; i < 10; i++ {
-		u := fmt.Sprintf("permanent%d", i)
-		permanentRelays = append(permanentRelays, &relay{URL: u})
-	}
-
-	knownRelays = []*relay{
-		{URL: "known1"},
-		{URL: "known2"},
-		{URL: "known3"},
-	}
-
-	mut = new(sync.RWMutex)
-}
-
-// Regression test: handleGetRequest should not modify permanentRelays.
-func TestHandleGetRequest(t *testing.T) {
-	needcap := len(permanentRelays) + len(knownRelays)
-	if needcap > cap(permanentRelays) {
-		t.Fatalf("test setup failed: need cap(permanentRelays) >= %d, have %d",
-			needcap, cap(permanentRelays))
-	}
-
-	w := httptest.NewRecorder()
-	w.Body = new(bytes.Buffer)
-	handleGetRequest(w, httptest.NewRequest("GET", "/", nil))
-
-	result := make(map[string][]*relay)
-	err := json.NewDecoder(w.Body).Decode(&result)
-	if err != nil {
-		t.Fatalf("invalid JSON: %v", err)
-	}
-
-	relays := result["relays"]
-	expect, actual := len(knownRelays)+len(permanentRelays), len(relays)
-	if actual != expect {
-		t.Errorf("expected %d relays, got %d", expect, actual)
-	}
-
-	// Check for changes in permanentRelays.
-	for i, r := range permanentRelays {
-		switch {
-		case !strings.HasPrefix(r.URL, "permanent"):
-			t.Errorf("relay %q among permanent relays", r.URL)
-		case r.URL != fmt.Sprintf("permanent%d", i):
-			t.Error("order of permanent relays changed")
-		}
-	}
-}
-
-func TestCanonicalizeQueryValues(t *testing.T) {
-	// This just demonstrates and validates the uri.Parse/String stuff in
-	// regards to query strings.
-
-	in := "http://example.com/?some weird= query^value"
-	exp := "http://example.com/?some+weird=+query%5Evalue"
-
-	uri, err := url.Parse(in)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	str := uri.String()
-	if str != in {
-		// Just re-encoding the URL doesn't sanitize the query string.
-		t.Errorf("expected %q, got %q", in, str)
-	}
-
-	uri.RawQuery = uri.Query().Encode()
-	str = uri.String()
-	if str != exp {
-		// The query string is now in correct format.
-		t.Errorf("expected %q, got %q", exp, str)
-	}
-}

cmd/strelaypoolsrv/stats.go

@@ -10,12 +10,11 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
 func init() {
-	processCollectorOpts := collectors.ProcessCollectorOpts{
+	processCollectorOpts := prometheus.ProcessCollectorOpts{
 		Namespace: "syncthing_relaypoolsrv",
 		PidFn: func() (int, error) {
 			return os.Getpid(), nil
@@ -23,7 +22,7 @@ func init() {
 	}
 
 	prometheus.MustRegister(
-		collectors.NewProcessCollector(processCollectorOpts),
+		prometheus.NewProcessCollector(processCollectorOpts),
 	)
 }
 
@@ -127,7 +126,7 @@ func refreshStats() {
 		go func(rel *relay) {
 			t0 := time.Now()
 			stats := fetchStats(rel)
-			duration := time.Since(t0).Seconds()
+			duration := time.Now().Sub(t0).Seconds()
 			result := "success"
 			if stats == nil {
 				result = "failed"

cmd/strelaysrv/README.md

@@ -121,11 +121,11 @@ Relay related libraries used by this repo
 ----
 ##### Relay protocol definition.
 
-[Available here](https://github.com/syncthing/syncthing/tree/main/lib/relay/protocol)
+[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/protocol)
 
 
 ##### Relay client
 
 Only used by the testutil.
 
-[Available here](https://github.com/syncthing/syncthing/tree/main/lib/relay/client)
+[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/client)

cmd/strelaysrv/etc/firewall-ufw/strelaysrv

@@ -1,9 +0,0 @@
-[strelaysrv]
-title=Syncthing relay server
-description=Proxies traffic of syncthing client behind firewalls
-ports=22067/tcp
-
-[strelaysrv-metrics]
-title=Syncthing relay metrics
-description=Provides metrics about the syncthing relay server
-ports=22070/tcp

cmd/strelaysrv/etc/linux-systemd/default

@@ -1,5 +0,0 @@
-# Default settings for syncthing-relaysrv (strelaysrv).
-NAT=true
-
-## Add Options here:
-RELAYSRV_OPTS=

cmd/strelaysrv/etc/linux-systemd/strelaysrv.service

@@ -1,25 +1,17 @@
 [Unit]
-Description=Syncthing Relay Server
+Description=Syncthing relay server
 After=network.target
-Documentation=man:strelaysrv(1)
 
 [Service]
-WorkingDirectory=/var/lib/syncthing-relaysrv
-EnvironmentFile=/etc/default/syncthing-relaysrv
-ExecStart=/usr/bin/strelaysrv -nat=${NAT} $RELAYSRV_OPTS
+User=strelaysrv
+Group=strelaysrv
+ExecStart=/usr/bin/strelaysrv
+WorkingDirectory=/var/lib/strelaysrv
 
-# Hardening
-User=syncthing-relaysrv
-Group=syncthing
-ProtectSystem=strict
-ReadWritePaths=/var/lib/syncthing-relaysrv
-NoNewPrivileges=true
 PrivateTmp=true
-PrivateDevices=true
+ProtectSystem=full
 ProtectHome=true
-SystemCallArchitectures=native
-MemoryDenyWriteExecute=true
+NoNewPrivileges=true
 
 [Install]
 WantedBy=multi-user.target
-Alias=syncthing-relaysrv.service

cmd/strelaysrv/listener.go

@@ -20,10 +20,10 @@ import (
 var (
 	outboxesMut    = sync.RWMutex{}
 	outboxes       = make(map[syncthingprotocol.DeviceID]chan interface{})
-	numConnections atomic.Int64
+	numConnections int64
 )
 
-func listener(_, addr string, config *tls.Config, token string) {
+func listener(proto, addr string, config *tls.Config) {
 	tcpListener, err := net.Listen("tcp", addr)
 	if err != nil {
 		log.Fatalln(err)
@@ -36,14 +36,8 @@ func listener(_, addr string, config *tls.Config, token string) {
 	for {
 		conn, isTLS, err := listener.AcceptNoWrapTLS()
 		if err != nil {
-			// Conn may be nil if accept failed, or non-nil if the initial
-			// read to figure out if it's TLS or not failed. In the latter
-			// case, close the connection before moving on.
-			if conn != nil {
-				conn.Close()
-			}
 			if debug {
-				log.Println("Listener failed to accept:", err)
+				log.Println("Listener failed to accept connection from", conn.RemoteAddr(), ". Possibly a TCP Ping.")
 			}
 			continue
 		}
@@ -55,7 +49,7 @@ func listener(_, addr string, config *tls.Config, token string) {
 		}
 
 		if isTLS {
-			go protocolConnectionHandler(conn, config, token)
+			go protocolConnectionHandler(conn, config)
 		} else {
 			go sessionConnectionHandler(conn)
 		}
@@ -63,7 +57,7 @@ func listener(_, addr string, config *tls.Config, token string) {
 	}
 }
 
-func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token string) {
+func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
 	conn := tls.Server(tcpConn, config)
 	if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil {
 		if debug {
@@ -82,7 +76,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token strin
 	}
 
 	state := conn.ConnectionState()
-	if debug && state.NegotiatedProtocol != protocol.ProtocolName {
+	if (!state.NegotiatedProtocolIsMutual || state.NegotiatedProtocol != protocol.ProtocolName) && debug {
 		log.Println("Protocol negotiation error")
 	}
 
@@ -125,16 +119,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token strin
 
 			switch msg := message.(type) {
 			case protocol.JoinRelayRequest:
-				if token != "" && msg.Token != token {
-					if debug {
-						log.Printf("invalid token %s\n", msg.Token)
-					}
-					protocol.WriteMessage(conn, protocol.ResponseWrongToken)
-					conn.Close()
-					continue
-				}
-
-				if overLimit.Load() {
+				if atomic.LoadInt32(&overLimit) > 0 {
 					protocol.WriteMessage(conn, protocol.RelayFull{})
 					if debug {
 						log.Println("Refusing join request from", id, "due to being over limits")
@@ -164,15 +149,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token strin
 				protocol.WriteMessage(conn, protocol.ResponseSuccess)
 
 			case protocol.ConnectRequest:
-				requestedPeer, err := syncthingprotocol.DeviceIDFromBytes(msg.ID)
-				if err != nil {
-					if debug {
-						log.Println(id, "is looking for an invalid peer ID")
-					}
-					protocol.WriteMessage(conn, protocol.ResponseNotFound)
-					conn.Close()
-					continue
-				}
+				requestedPeer := syncthingprotocol.DeviceIDFromBytes(msg.ID)
 				outboxesMut.RLock()
 				peerOutbox, ok := outboxes[requestedPeer]
 				outboxesMut.RUnlock()
@@ -273,7 +250,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token strin
 				conn.Close()
 			}
 
-			if overLimit.Load() && !hasSessions(id) {
+			if atomic.LoadInt32(&overLimit) > 0 && !hasSessions(id) {
 				if debug {
 					log.Println("Dropping", id, "as it has no sessions and we are over our limits")
 				}
@@ -366,8 +343,8 @@ func sessionConnectionHandler(conn net.Conn) {
 }
 
 func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) {
-	numConnections.Add(1)
-	defer numConnections.Add(-1)
+	atomic.AddInt64(&numConnections, 1)
+	defer atomic.AddInt64(&numConnections, -1)
 
 	for {
 		msg, err := protocol.ReadMessage(conn)

cmd/strelaysrv/main.go

@@ -3,7 +3,6 @@
 package main
 
 import (
-	"context"
 	"crypto/tls"
 	"flag"
 	"fmt"
@@ -14,13 +13,13 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"runtime"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"syscall"
 	"time"
 
-	"github.com/syncthing/syncthing/lib/build"
-	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/relay/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -34,6 +33,24 @@ import (
 	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
 )
 
+var (
+	Version    string
+	BuildStamp string
+	BuildUser  string
+	BuildHost  string
+
+	BuildDate   time.Time
+	LongVersion string
+)
+
+func init() {
+	stamp, _ := strconv.Atoi(BuildStamp)
+	BuildDate = time.Unix(int64(stamp), 0)
+
+	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
+	LongVersion = fmt.Sprintf(`strelaysrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
+}
+
 var (
 	listen string
 	debug  bool
@@ -49,14 +66,13 @@ var (
 
 	sessionLimitBps   int
 	globalLimitBps    int
-	overLimit         atomic.Bool
+	overLimit         int32
 	descriptorLimit   int64
 	sessionLimiter    *rate.Limiter
 	globalLimiter     *rate.Limiter
 	networkBufferSize int
 
 	statusAddr       string
-	token            string
 	poolAddrs        string
 	pools            []string
 	providedBy       string
@@ -90,7 +106,6 @@ func main() {
 	flag.IntVar(&globalLimitBps, "global-rate", globalLimitBps, "Global rate limit, in bytes/s")
 	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
 	flag.StringVar(&statusAddr, "status-srv", ":22070", "Listen address for status service (blank to disable)")
-	flag.StringVar(&token, "token", "", "Token to restrict access to the relay (optional). Disables joining any pools.")
 	flag.StringVar(&poolAddrs, "pools", defaultPoolAddrs, "Comma separated list of relay pool addresses to join")
 	flag.StringVar(&providedBy, "provided-by", "", "An optional description about who provides the relay")
 	flag.StringVar(&extAddress, "ext-address", "", "An optional address to advertise as being available on.\n\tAllows listening on an unprivileged port with port forwarding from e.g. 443, and be connected to on port 443.")
@@ -100,16 +115,9 @@ func main() {
 	flag.IntVar(&natRenewal, "nat-renewal", 30, "NAT renewal frequency in minutes")
 	flag.IntVar(&natTimeout, "nat-timeout", 10, "NAT discovery timeout in seconds")
 	flag.BoolVar(&pprofEnabled, "pprof", false, "Enable the built in profiling on the status server")
-	flag.IntVar(&networkBufferSize, "network-buffer", 65536, "Network buffer size (two of these per proxied connection)")
-	showVersion := flag.Bool("version", false, "Show version")
+	flag.IntVar(&networkBufferSize, "network-buffer", 2048, "Network buffer size (two of these per proxied connection)")
 	flag.Parse()
 
-	longVer := build.LongVersionFor("strelaysrv")
-	if *showVersion {
-		fmt.Println(longVer)
-		return
-	}
-
 	if extAddress == "" {
 		extAddress = listen
 	}
@@ -138,7 +146,7 @@ func main() {
 		}
 	}
 
-	log.Println(longVer)
+	log.Println(LongVersion)
 
 	maxDescriptors, err := osutil.MaximizeOpenFileLimit()
 	if maxDescriptors > 0 {
@@ -147,7 +155,7 @@ func main() {
 		log.Println("Connection limit", descriptorLimit)
 
 		go monitorLimits()
-	} else if err != nil && !build.IsWindows {
+	} else if err != nil && runtime.GOOS != "windows" {
 		log.Println("Assuming no connection limit, due to error retrieving rlimits:", err)
 	}
 
@@ -158,7 +166,7 @@ func main() {
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 20*365)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv")
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
@@ -186,22 +194,19 @@ func main() {
 		log.Println("ID:", id)
 	}
 
-	wrapper := config.Wrap("config", config.New(id), id, events.NoopLogger)
-	go wrapper.Serve(context.TODO())
-	wrapper.Modify(func(cfg *config.Configuration) {
-		cfg.Options.NATLeaseM = natLease
-		cfg.Options.NATRenewalM = natRenewal
-		cfg.Options.NATTimeoutS = natTimeout
+	wrapper := config.Wrap("config", config.New(id))
+	wrapper.SetOptions(config.OptionsConfiguration{
+		NATLeaseM:   natLease,
+		NATRenewalM: natRenewal,
+		NATTimeoutS: natTimeout,
 	})
 	natSvc := nat.NewService(id, wrapper)
 	mapping := mapping{natSvc.NewMapping(nat.TCP, addr.IP, addr.Port)}
 
 	if natEnabled {
-		ctx, cancel := context.WithCancel(context.Background())
-		go natSvc.Serve(ctx)
-		defer cancel()
+		go natSvc.Serve()
 		found := make(chan struct{})
-		mapping.OnChanged(func() {
+		mapping.OnChanged(func(_ *nat.Mapping, _, _ []nat.Address) {
 			select {
 			case found <- struct{}{}:
 			default:
@@ -231,37 +236,13 @@ func main() {
 		go statusService(statusAddr)
 	}
 
-	uri, err := url.Parse(fmt.Sprintf("relay://%s/", mapping.Address()))
+	uri, err := url.Parse(fmt.Sprintf("relay://%s/?id=%s&pingInterval=%s&networkTimeout=%s&sessionLimitBps=%d&globalLimitBps=%d&statusAddr=%s&providedBy=%s", mapping.Address(), id, pingInterval, networkTimeout, sessionLimitBps, globalLimitBps, statusAddr, providedBy))
 	if err != nil {
 		log.Fatalln("Failed to construct URI", err)
-		return
 	}
 
-	// Add properly encoded query string parameters to URL.
-	query := make(url.Values)
-	query.Set("id", id.String())
-	query.Set("pingInterval", pingInterval.String())
-	query.Set("networkTimeout", networkTimeout.String())
-	if sessionLimitBps > 0 {
-		query.Set("sessionLimitBps", fmt.Sprint(sessionLimitBps))
-	}
-	if globalLimitBps > 0 {
-		query.Set("globalLimitBps", fmt.Sprint(globalLimitBps))
-	}
-	if statusAddr != "" {
-		query.Set("statusAddr", statusAddr)
-	}
-	if providedBy != "" {
-		query.Set("providedBy", providedBy)
-	}
-	uri.RawQuery = query.Encode()
-
 	log.Println("URI:", uri.String())
 
-	if token != "" {
-		poolAddrs = ""
-	}
-
 	if poolAddrs == defaultPoolAddrs {
 		log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
 		log.Println("!!  Joining default relay pools, this relay will be available for public use. !!")
@@ -273,11 +254,11 @@ func main() {
 	for _, pool := range pools {
 		pool = strings.TrimSpace(pool)
 		if len(pool) > 0 {
-			go poolHandler(pool, uri, mapping, cert)
+			go poolHandler(pool, uri, mapping)
 		}
 	}
 
-	go listener(proto, listen, tlsCfg, token)
+	go listener(proto, listen, tlsCfg)
 
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
@@ -308,10 +289,10 @@ func main() {
 func monitorLimits() {
 	limitCheckTimer = time.NewTimer(time.Minute)
 	for range limitCheckTimer.C {
-		if numConnections.Load()+numProxies.Load() > descriptorLimit {
-			overLimit.Store(true)
+		if atomic.LoadInt64(&numConnections)+atomic.LoadInt64(&numProxies) > descriptorLimit {
+			atomic.StoreInt32(&overLimit, 1)
 			log.Println("Gone past our connection limits. Starting to refuse new/drop idle connections.")
-		} else if overLimit.CompareAndSwap(true, false) {
+		} else if atomic.CompareAndSwapInt32(&overLimit, 1, 0) {
 			log.Println("Dropped below our connection limits. Accepting new connections.")
 		}
 		limitCheckTimer.Reset(time.Minute)

cmd/strelaysrv/pool.go

@@ -4,20 +4,14 @@ package main
 
 import (
 	"bytes"
-	"crypto/tls"
 	"encoding/json"
-	"io"
+	"io/ioutil"
 	"log"
-	"net/http"
 	"net/url"
 	"time"
 )
 
-const (
-	httpStatusEnhanceYourCalm = 429
-)
-
-func poolHandler(pool string, uri *url.URL, mapping mapping, ownCert tls.Certificate) {
+func poolHandler(pool string, uri *url.URL, mapping mapping) {
 	if debug {
 		log.Println("Joining", pool)
 	}
@@ -32,80 +26,40 @@ func poolHandler(pool string, uri *url.URL, mapping mapping, ownCert tls.Certifi
 			uriCopy.String(),
 		})
 
-		poolUrl, err := url.Parse(pool)
+		resp, err := httpClient.Post(pool, "application/json", &b)
 		if err != nil {
-			log.Printf("Could not parse pool url '%s': %v", pool, err)
-		}
-
-		client := http.DefaultClient
-		if poolUrl.Scheme == "https" {
-			// Sent our certificate in join request
-			client = &http.Client{
-				Transport: &http.Transport{
-					TLSClientConfig: &tls.Config{
-						Certificates: []tls.Certificate{ownCert},
-					},
-				},
+			log.Println("Error joining pool", pool, err)
+		} else if resp.StatusCode == 500 {
+			bs, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				log.Println("Failed to join", pool, "due to an internal server error. Could not read response:", err)
+			} else {
+				log.Println("Failed to join", pool, "due to an internal server error:", string(bs))
 			}
-		}
-
-		resp, err := client.Post(pool, "application/json", &b)
-		if err != nil {
-			log.Printf("Error joining pool %v: HTTP request: %v", pool, err)
+			resp.Body.Close()
+		} else if resp.StatusCode == 429 {
+			log.Println(pool, "under load, will retry in a minute")
 			time.Sleep(time.Minute)
 			continue
-		}
-
-		bs, err := io.ReadAll(resp.Body)
-		resp.Body.Close()
-		if err != nil {
-			log.Printf("Error joining pool %v: reading response: %v", pool, err)
-			time.Sleep(time.Minute)
-			continue
-		}
-
-		switch resp.StatusCode {
-		case http.StatusOK:
+		} else if resp.StatusCode == 401 {
+			log.Println(pool, "failed to join due to IP address not matching external address. Aborting")
+			return
+		} else if resp.StatusCode == 200 {
 			var x struct {
 				EvictionIn time.Duration `json:"evictionIn"`
 			}
-			if err := json.Unmarshal(bs, &x); err == nil {
+			err := json.NewDecoder(resp.Body).Decode(&x)
+			if err == nil {
 				rejoin := x.EvictionIn - (x.EvictionIn / 5)
-				log.Printf("Joined pool %s, rejoining in %v", pool, rejoin)
+				log.Println("Joined", pool, "rejoining in", rejoin)
 				time.Sleep(rejoin)
 				continue
+			} else {
+				log.Println("Failed to deserialize response", err)
 			}
-			log.Printf("Joined pool %s, failed to deserialize response: %v", pool, err)
-
-		case http.StatusInternalServerError:
-			log.Printf("Failed to join %v: server error", pool)
-			log.Printf("Response data: %s", bs)
-			time.Sleep(time.Minute)
-			continue
-
-		case http.StatusBadRequest:
-			log.Printf("Failed to join %v: request or check error", pool)
-			log.Printf("Response data: %s", bs)
-			time.Sleep(time.Minute)
-			continue
-
-		case httpStatusEnhanceYourCalm:
-			log.Printf("Failed to join %v: under load (rate limiting)", pool)
-			time.Sleep(time.Minute)
-			continue
-
-		case http.StatusUnauthorized:
-			log.Printf("Failed to join %v: IP address not matching external address", pool)
-			log.Println("Aborting")
-			return
-
-		default:
-			log.Printf("Failed to join %v: unexpected status code from server: %d", pool, resp.StatusCode)
-			log.Printf("Response data: %s", bs)
-			time.Sleep(time.Minute)
-			continue
+		} else {
+			log.Println(pool, "unknown response type from server", resp.StatusCode)
 		}
-
 		time.Sleep(time.Hour)
 	}
 }

cmd/strelaysrv/scripts/preinst

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-addgroup --system syncthing
-adduser --system --home /var/lib/syncthing-relaysrv --ingroup syncthing syncthing-relaysrv

cmd/strelaysrv/session.go

@@ -22,9 +22,9 @@ import (
 var (
 	sessionMut      = sync.RWMutex{}
 	activeSessions  = make([]*session, 0)
-	pendingSessions = make(map[string]*session)
-	numProxies      atomic.Int64
-	bytesProxied    atomic.Int64
+	pendingSessions = make(map[string]*session, 0)
+	numProxies      int64
+	bytesProxied    int64
 )
 
 func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionRateLimit, globalRateLimit *rate.Limiter) *session {
@@ -251,8 +251,8 @@ func (s *session) proxy(c1, c2 net.Conn) error {
 		log.Println("Proxy", c1.RemoteAddr(), "->", c2.RemoteAddr())
 	}
 
-	numProxies.Add(1)
-	defer numProxies.Add(-1)
+	atomic.AddInt64(&numProxies, 1)
+	defer atomic.AddInt64(&numProxies, -1)
 
 	buf := make([]byte, networkBufferSize)
 	for {
@@ -262,7 +262,7 @@ func (s *session) proxy(c1, c2 net.Conn) error {
 			return err
 		}
 
-		bytesProxied.Add(int64(n))
+		atomic.AddInt64(&bytesProxied, int64(n))
 
 		if debug {
 			log.Printf("%d bytes from %s to %s", n, c1.RemoteAddr(), c2.RemoteAddr())

cmd/strelaysrv/status.go

@@ -10,8 +10,6 @@ import (
 	"runtime"
 	"sync/atomic"
 	"time"
-
-	"github.com/syncthing/syncthing/lib/build"
 )
 
 var rc *rateCalculator
@@ -36,24 +34,24 @@ func statusService(addr string) {
 	}
 }
 
-func getStatus(w http.ResponseWriter, _ *http.Request) {
+func getStatus(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Access-Control-Allow-Origin", "*")
 	status := make(map[string]interface{})
 
 	sessionMut.Lock()
 	// This can potentially be double the number of pending sessions, as each session has two keys, one for each side.
-	status["version"] = build.Version
-	status["buildHost"] = build.Host
-	status["buildUser"] = build.User
-	status["buildDate"] = build.Date
+	status["version"] = Version
+	status["buildHost"] = BuildHost
+	status["buildUser"] = BuildUser
+	status["buildDate"] = BuildDate
 	status["startTime"] = rc.startTime
 	status["uptimeSeconds"] = time.Since(rc.startTime) / time.Second
 	status["numPendingSessionKeys"] = len(pendingSessions)
 	status["numActiveSessions"] = len(activeSessions)
 	sessionMut.Unlock()
-	status["numConnections"] = numConnections.Load()
-	status["numProxies"] = numProxies.Load()
-	status["bytesProxied"] = bytesProxied.Load()
+	status["numConnections"] = atomic.LoadInt64(&numConnections)
+	status["numProxies"] = atomic.LoadInt64(&numProxies)
+	status["bytesProxied"] = atomic.LoadInt64(&bytesProxied)
 	status["goVersion"] = runtime.Version()
 	status["goOS"] = runtime.GOOS
 	status["goArch"] = runtime.GOARCH
@@ -88,13 +86,13 @@ func getStatus(w http.ResponseWriter, _ *http.Request) {
 }
 
 type rateCalculator struct {
-	counter   *atomic.Int64
 	rates     []int64
 	prev      int64
+	counter   *int64
 	startTime time.Time
 }
 
-func newRateCalculator(keepIntervals int, interval time.Duration, counter *atomic.Int64) *rateCalculator {
+func newRateCalculator(keepIntervals int, interval time.Duration, counter *int64) *rateCalculator {
 	r := &rateCalculator{
 		rates:     make([]int64, keepIntervals),
 		counter:   counter,
@@ -112,7 +110,7 @@ func (r *rateCalculator) updateRates(interval time.Duration) {
 		next := now.Truncate(interval).Add(interval)
 		time.Sleep(next.Sub(now))
 
-		cur := r.counter.Load()
+		cur := atomic.LoadInt64(r.counter)
 		rate := int64(float64(cur-r.prev) / interval.Seconds())
 		copy(r.rates[1:], r.rates)
 		r.rates[0] = rate