lib/model, lib/versioner: Prevent symlink attack via versioning (fixes #4286)

Prior to this, the following is possible:

- Create a symlink "foo -> /somewhere", it gets synced
- Delete "foo", it gets versioned
- Create "foo/bar", it gets synced
- Delete "foo/bar", it gets versioned in "/somewhere/bar"

With this change, versioners should never version symlinks.

.codecov.yml

@@ -1,20 +0,0 @@
-comment: false
-
-coverage:
-  range: "40...100"
-  precision: 1
-  status:
-    patch:
-      default:
-        informational: true
-    project:
-      default:
-        informational: true
-
-github_checks:
-  annotations: false
-
-ignore:
-  - "**.pb.go"
-  - "**_mocked.go"
-  - "**/mocks/*"

.deepsource.toml

@@ -1,12 +0,0 @@
-version = 1
-
-exclude_patterns = ["**/*.pb.go"]
-test_patterns = ["**/*_test.go"]
-
-[[analyzers]]
-name = "go"
-enabled = true
-
-  [analyzers.meta]
-  import_paths = ["github.com/syncthing/syncthing"]
-  build_tags = ["noassets"]

.github/CODEOWNERS

@@ -1,2 +0,0 @@
-/AUTHORS    @calmh
-/*.md       @calmh

.github/FUNDING.yml

@@ -1,11 +0,0 @@
-github: syncthing
-custom: "https://syncthing.net/donations/"
-
-# patreon: # Replace with a single Patreon username
-# open_collective: # Replace with a single Open Collective username
-# ko_fi: # Replace with a single Ko-fi username
-# tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-# community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-# liberapay: # Replace with a single Liberapay username
-# issuehunt: # Replace with a single IssueHunt username
-# otechie: # Replace with a single Otechie username

.github/ISSUE_TEMPLATE/01-feature.md

@@ -1,13 +0,0 @@
----
-name: Feature request
-about: If you're just not sure how to do something, see "ask a question".
-labels: enhancement, needs-triage
----
-
-### Include required information
-
-Please be sure to include at least:
-
- - what problem your new feature would solve
- - how or why you think it is generally useful (i.e., not just for you)
- - what alternatives or workarounds you considered

.github/ISSUE_TEMPLATE/02-bug.md

@@ -1,23 +0,0 @@
----
-name: Bug report
-about: If you're actually looking for support, see "ask a question".
-labels: bug, needs-triage
----
-
-### Does your log mention database corruption?
-
-If your Syncthing log reports panics because of database corruption it is
-most likely a fault with your system's storage or memory. Affected log
-entries will contain lines starting with `panic: leveldb`. You will need to
-delete the index database to clear this, by running `syncthing
--reset-database`.
-
-### Include required information
-
-Please be sure to include at least:
-
- - which version of Syncthing and what operating system you are using
- - browser and version, if applicable
- - what happened,
- - what you expected to happen instead, and
- - any steps to reproduce the problem.

.github/ISSUE_TEMPLATE/config.yml

@@ -1,8 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: I need help / I have a question
-    url: https://forum.syncthing.net/
-    about: Ask questions, get support, and discuss with other community members.
-  - name: Android issues
-    url: https://github.com/syncthing/syncthing-android/issues/
-    about: The Android app has its own issue tracker.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,27 +0,0 @@
-### Purpose
-
-Describe the purpose of this change. If there is an existing issue that is
-resolved by this pull request, ensure that the commit subject is on the form
-`Some short description (fixes #1234)` where 1234 is the issue number.
-
-### Testing
-
-Describe what testing has been done, and how the reviewer can test the change
-if new tests are not included.
-
-### Screenshots
-
-If this is a GUI change, include screenshots of the change. If not, please
-feel free to just delete this section.
-
-### Documentation
-
-If this is a user visible change (including API and protocol changes), add a link here
-to the corresponding pull request on https://github.com/syncthing/docs or describe
-the documentation changes necessary.
-
-## Authorship
-
-Your name and email will be added automatically to the AUTHORS file
-based on the commit metadata.
-

.github/SECURITY.md

@@ -1,10 +0,0 @@
-## Reporting a Vulnerability
-
-If you believe that you've found a Syncthing-related security vulnerability,
-please report it by sending email to the address security@syncthing.net. The
-[PGP key for security@syncthing.net
-(B683AD7B76CAB013)](https://syncthing.net/security-key.txt) can be used to
-send encrypted mail or to verify responses received from that address.
-
-You can read more about Syncthing security at
-https://syncthing.net/security/.

.github/dependabot.yml

@@ -1,13 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: weekly
-    open-pull-requests-limit: 10
-
-  - package-ecosystem: "gomod"
-    directory: "/"
-    schedule:
-      interval: weekly
-    open-pull-requests-limit: 10

.github/workflows/build-infra-dockers.yaml

@@ -1,54 +0,0 @@
-name: Build Infrastructure Images
-
-on:
-  push:
-    branches:
-      - infrastructure
-
-env:
-    GO_VERSION: "~1.21.1"
-    CGO_ENABLED: "0"
-    BUILD_USER: docker
-    BUILD_HOST: github.syncthing.net
-
-jobs:
-
-  docker-syncthing:
-    name: Build and push Docker images
-    runs-on: ubuntu-latest
-    environment: docker
-    strategy:
-        matrix:
-            pkg:
-                - stcrashreceiver
-                - strelaypoolsrv
-                - stupgrades
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Build binaries
-        run: |
-          for arch in arm64 amd64; do
-            go run build.go -goos linux -goarch "$arch" build ${{ matrix.pkg }}
-            mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-"$arch"
-          done
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./Dockerfile.${{ matrix.pkg }}
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: syncthing/${{ matrix.pkg }}:latest,syncthing/${{ matrix.pkg }}:${{ github.sha }}

.github/workflows/build-syncthing.yaml

@@ -1,795 +0,0 @@
-name: Build Syncthing
-
-on:
-  pull_request:
-  push:
-  schedule:
-    # Run nightly build at 05:00 UTC
-    - cron: '00 05 * * *'
-  workflow_dispatch:
-
-env:
-  # The go version to use for builds. We set check-latest to true when
-  # installing, so we get the latest patch version that matches the
-  # expression.
-  GO_VERSION: "~1.21.1"
-
-  # Optimize compatibility on the slow archictures.
-  GO386: softfloat
-  GOARM: "5"
-  GOMIPS: softfloat
-
-  # Avoid hilarious amounts of obscuring log output when running tests.
-  LOGGER_DISCARD: "1"
-
-  # Our build metadata
-  BUILD_USER: builder
-  BUILD_HOST: github.syncthing.net
-
-# A note on actions and third party code... The actions under actions/ (like
-# `uses: actions/checkout`) are maintained by GitHub, and we need to trust
-# GitHub to maintain their code and infrastructure or we're in deep shit in
-# general. The same doesn't necessarily apply to other actions authors, so
-# some care needs to be taken when adding steps, especially in the paths
-# that lead up to code being packaged and signed.
-
-jobs:
-
-  #
-  # Tests for all platforms. Runs a matrix build on Windows, Linux and Mac,
-  # with the list of expected supported Go versions (current, previous).
-  #
-
-  build-test:
-    name: Build and test
-    strategy:
-      fail-fast: false
-      matrix:
-        runner: ["windows-latest", "ubuntu-latest", "macos-latest"]
-        # The oldest version in this list should match what we have in our go.mod.
-        # Variables don't seem to be supported here, or we could have done something nice.
-        go: ["1.20", "1.21"]
-    runs-on: ${{ matrix.runner }}
-    steps:
-      - name: Set git to use LF
-        if: matrix.runner == 'windows-latest'
-        # Without this, the Windows checkout will happen with CRLF line
-        # endings, which is fine for the source code but messes up tests
-        # that depend on data on disk being as expected. Ideally, those
-        # tests should be fixed, but not today.
-        run: |
-          git config --global core.autocrlf false
-          git config --global core.eol lf
-
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ matrix.go }}
-          cache: true
-          check-latest: true
-
-      - name: Build
-        run: |
-          go run build.go
-
-      - name: Install go-test-json-to-loki
-        run: |
-          go install calmh.dev/go-test-json-to-loki@latest
-
-      - name: Test
-        run: |
-          go run build.go test | go-test-json-to-loki
-        env:
-          GOFLAGS: "-json"
-          LOKI_URL: ${{ secrets.LOKI_URL }}
-          LOKI_USER: ${{ secrets.LOKI_USER }}
-          LOKI_PASSWORD: ${{ secrets.LOKI_PASSWORD }}
-          LOKI_LABELS: "go=${{ matrix.go }},runner=${{ matrix.runner }},repo=${{ github.repository }},ref=${{ github.ref }}"
-
-  #
-  # Meta checks for formatting, copyright, etc
-  #
-
-  correctness:
-    name: Check correctness
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Check correctness
-        run: |
-          go test -v ./meta
-
-  #
-  # The basic checks job is a virtual one that depends on the matrix tests,
-  # the correctness checks, and various builds that we always do. This makes
-  # it easy to have the PR process have a single test as a gatekeeper for
-  # merging, instead of having to add all the matrix tests and update them
-  # each time the version changes. (The top level test is not available for
-  # choosing there, only the matrix "children".)
-  #
-
-  basics:
-    name: Basic checks passed
-    runs-on: ubuntu-latest
-    needs:
-      - build-test
-      - correctness
-      - package-linux
-      - package-cross
-      - package-source
-      - package-debian
-      - govulncheck
-    steps:
-      - uses: actions/checkout@v4
-
-  #
-  # Windows
-  #
-
-  package-windows:
-    name: Package for Windows
-    if: github.event_name == 'push' && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
-    environment: signing
-    runs-on: windows-latest
-    steps:
-      - name: Set git to use LF
-        # Without this, the checkout will happen with CRLF line endings,
-        # which is fine for the source code but messes up tests that depend
-        # on data on disk being as expected. Ideally, those tests should be
-        # fixed, but not today.
-        run: |
-          git config --global core.autocrlf false
-          git config --global core.eol lf
-
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~\AppData\Local\go-build
-            ~\go\pkg\mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
-
-      - name: Install dependencies
-        run: |
-          go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@v1.4.0
-
-      - name: Create packages
-        run: |
-          go run build.go -goarch amd64 zip
-          go run build.go -goarch arm zip
-          go run build.go -goarch arm64 zip
-          go run build.go -goarch 386 zip
-        env:
-          CGO_ENABLED: "0"
-          CODESIGN_SIGNTOOL: ${{ secrets.CODESIGN_SIGNTOOL }}
-          CODESIGN_CERTIFICATE_BASE64: ${{ secrets.CODESIGN_CERTIFICATE_BASE64 }}
-          CODESIGN_CERTIFICATE_PASSWORD: ${{ secrets.CODESIGN_CERTIFICATE_PASSWORD }}
-          CODESIGN_TIMESTAMP_SERVER: ${{ secrets.CODESIGN_TIMESTAMP_SERVER }}
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: packages-windows
-          path: syncthing-windows-*.zip
-
-  #
-  # Linux
-  #
-
-  package-linux:
-    name: Package for Linux
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
-
-      - name: Create packages
-        run: |
-          archs=$(go tool dist list | grep linux | sed 's#linux/##')
-          for goarch in $archs ; do
-            go run build.go -goarch "$goarch" tar
-          done
-        env:
-          CGO_ENABLED: "0"
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: packages-linux
-          path: syncthing-linux-*.tar.gz
-
-  #
-  # macOS
-  #
-
-  package-macos:
-    name: Package for macOS
-    if: github.event_name == 'push' && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
-    environment: signing
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
-
-      - name: Import signing certificate
-        run: |
-          # Set up a run-specific keychain, making it available for the
-          # `codesign` tool.
-          umask 066
-          KEYCHAIN_PATH=$RUNNER_TEMP/codesign.keychain
-          KEYCHAIN_PASSWORD=$(uuidgen)
-          security create-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
-          security default-keychain -s "$KEYCHAIN_PATH"
-          security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
-          security set-keychain-settings -lut 21600 "$KEYCHAIN_PATH"
-
-          # Import the certificate
-          CERTIFICATE_PATH=$RUNNER_TEMP/codesign.p12
-          echo "$DEVELOPER_ID_CERTIFICATE_BASE64" | base64 -d -o "$CERTIFICATE_PATH"
-          security import "$CERTIFICATE_PATH" -k "$KEYCHAIN_PATH" -P "$DEVELOPER_ID_CERTIFICATE_PASSWORD" -T /usr/bin/codesign -T /usr/bin/productsign
-          security set-key-partition-list -S apple-tool:,apple: -s -k actions "$KEYCHAIN_PATH"
-
-          # Set the codesign identity for following steps
-          echo "CODESIGN_IDENTITY=$CODESIGN_IDENTITY" >> $GITHUB_ENV
-        env:
-          DEVELOPER_ID_CERTIFICATE_BASE64: ${{ secrets.DEVELOPER_ID_CERTIFICATE_BASE64 }}
-          DEVELOPER_ID_CERTIFICATE_PASSWORD: ${{ secrets.DEVELOPER_ID_CERTIFICATE_PASSWORD }}
-          CODESIGN_IDENTITY: ${{ secrets.CODESIGN_IDENTITY }}
-
-      - name: Create package (amd64)
-        run: |
-          go run build.go -goarch amd64 zip
-        env:
-          CGO_ENABLED: "1"
-
-      - name: Create package (arm64 cross)
-        run: |
-          cat <<EOT > xgo.sh
-          #!/bin/bash
-          CGO_ENABLED=1 \
-            CGO_CFLAGS="-target arm64-apple-macos10.15" \
-            CGO_LDFLAGS="-target arm64-apple-macos10.15" \
-            go "\$@"
-          EOT
-          chmod 755 xgo.sh
-          go run build.go -gocmd ./xgo.sh -goarch arm64 zip
-        env:
-          CGO_ENABLED: "1"
-
-      - name: Create package (universal)
-        run: |
-          rm -rf _tmp
-          mkdir _tmp
-          pushd _tmp
-
-          unzip ../syncthing-macos-amd64-*.zip
-          unzip ../syncthing-macos-arm64-*.zip
-          lipo -create syncthing-macos-amd64-*/syncthing syncthing-macos-arm64-*/syncthing -o syncthing
-
-          amd64=(syncthing-macos-amd64-*)
-          universal="${amd64/amd64/universal}"
-          mv "$amd64" "$universal"
-          mv syncthing "$universal"
-          zip -r "../$universal.zip" "$universal"
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: packages-macos
-          path: syncthing-*.zip
-
-  notarize-macos:
-    name: Notarize for macOS
-    if: github.event_name == 'push' && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
-    environment: signing
-    needs:
-      - package-macos
-      - basics
-    runs-on: macos-latest
-    steps:
-      - name: Download artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: packages-macos
-
-      - name: Notarize binaries
-        run: |
-          APPSTORECONNECT_API_KEY_PATH="$RUNNER_TEMP/apikey.p8"
-          echo "$APPSTORECONNECT_API_KEY" | base64 -d -o "$APPSTORECONNECT_API_KEY_PATH"
-          for file in syncthing-macos-*.zip ; do
-            xcrun notarytool submit \
-              -k "$APPSTORECONNECT_API_KEY_PATH" \
-              -d "$APPSTORECONNECT_API_KEY_ID" \
-              -i "$APPSTORECONNECT_API_KEY_ISSUER" \
-              $file
-          done
-        env:
-          APPSTORECONNECT_API_KEY: ${{ secrets.APPSTORECONNECT_API_KEY }}
-          APPSTORECONNECT_API_KEY_ID: ${{ secrets.APPSTORECONNECT_API_KEY_ID }}
-          APPSTORECONNECT_API_KEY_ISSUER: ${{ secrets.APPSTORECONNECT_API_KEY_ISSUER }}
-
-  #
-  # Cross compile other unixes
-  #
-
-  package-cross:
-    name: Package cross compiled
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-cross-${{ hashFiles('**/go.sum') }}
-
-      - name: Create packages
-        run: |
-          platforms=$(go tool dist list \
-            | grep -v aix/ppc64 \
-            | grep -v android/ \
-            | grep -v darwin/ \
-            | grep -v ios/ \
-            | grep -v js/ \
-            | grep -v linux/ \
-            | grep -v nacl/ \
-            | grep -v plan9/ \
-            | grep -v windows/ \
-            | grep -v /wasm \
-          )
-
-          # Build for each platform with errors silenced, because we expect
-          # some oddball platforms to fail. This avoids a bunch of errors in
-          # the GitHub Actions output, instead summarizing each build
-          # failure as a warning.
-          for plat in $platforms; do
-            goos="${plat%/*}"
-            goarch="${plat#*/}"
-            echo "::group ::$plat"
-            if ! go run build.go -goos "$goos" -goarch "$goarch" tar 2>/dev/null; then
-              echo "::warning ::Failed to build for $plat"
-            fi
-            echo "::endgroup::"
-          done
-        env:
-          CGO_ENABLED: "0"
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: packages-other
-          path: syncthing-*.tar.gz
-
-  #
-  # Source
-  #
-
-  package-source:
-    name: Package source code
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: Package source
-        run: |
-          version=$(go run build.go version)
-          echo "$version" > RELEASE
-
-          go mod vendor
-          go run build.go assets
-
-          cd ..
-
-          tar c -z -f "syncthing-source-$version.tar.gz" \
-            --exclude .git \
-            syncthing
-
-          mv "syncthing-source-$version.tar.gz" syncthing
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: packages-source
-          path: syncthing-source-*.tar.gz
-
-  #
-  # Sign binaries for auto upgrade, generate ASC signature files
-  #
-
-  sign-for-upgrade:
-    name: Sign for upgrade
-    if: github.event_name == 'push' && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
-    environment: signing
-    needs:
-      - basics
-      - package-windows
-      - package-linux
-      - package-macos
-      - package-cross
-      - package-source
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/checkout@v4
-        with:
-          repository: syncthing/release-tools
-          path: tools
-          fetch-depth: 0
-
-      - name: Download artifacts
-        uses: actions/download-artifact@v3
-
-      - name: Install signing tool
-        run: |
-          go install ./cmd/stsigtool
-
-      - name: Sign archives
-        run: |
-          export PRIVATE_KEY="$RUNNER_TEMP/privkey.pem"
-          export PATH="$PATH:$(go env GOPATH)/bin"
-          echo "$STSIGTOOL_PRIVATE_KEY" | base64 -d > "$PRIVATE_KEY"
-          mkdir packages
-          mv packages-*/* packages
-          pushd packages
-          "$GITHUB_WORKSPACE/tools/sign-only"
-          rm -f "$PRIVATE_KEY"
-        env:
-          STSIGTOOL_PRIVATE_KEY: ${{ secrets.STSIGTOOL_PRIVATE_KEY }}
-
-      - name: Create and sign .asc files
-        run: |
-          sudo apt update
-          sudo apt -y install gnupg
-
-          export SIGNING_KEY="$RUNNER_TEMP/gpg-secret.asc"
-          echo "$GNUPG_SIGNING_KEY_BASE64" | base64 -d > "$SIGNING_KEY"
-          gpg --import < "$SIGNING_KEY"
-
-          pushd packages
-          files=(*.tar.gz *.zip)
-          sha1sum "${files[@]}" | gpg --clearsign > sha1sum.txt.asc
-          sha256sum "${files[@]}" | gpg --clearsign > sha256sum.txt.asc
-          gpg --sign --armour --detach syncthing-source-*.tar.gz
-          popd
-          rm -f "$SIGNING_KEY" .gnupg
-        env:
-          GNUPG_SIGNING_KEY_BASE64: ${{ secrets.GNUPG_SIGNING_KEY_BASE64 }}
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: packages-signed
-          path: packages/*
-
-  #
-  # Debian
-  #
-
-  package-debian:
-    name: Package for Debian
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: '3.0'
-
-      - name: Install fpm
-        run: |
-          gem install fpm
-
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-debian-${{ hashFiles('**/go.sum') }}
-
-      - name: Package for Debian
-        run: |
-          for arch in amd64 i386 armhf armel arm64 ; do
-            go run build.go -no-upgrade -installsuffix=no-upgrade -goarch "$arch" deb
-          done
-        env:
-          BUILD_USER: debian
-
-      - name: Archive artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: debian-packages
-          path: "*.deb"
-
-  #
-  # Nightlies
-  #
-
-  publish-nightly:
-    name: Publish nightly build
-    if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads/release-nightly')
-    environment: signing
-    needs:
-      - sign-for-upgrade
-      - notarize-macos
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: syncthing/release-tools
-          path: tools
-          fetch-depth: 0
-
-      - name: Download artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: packages-signed
-          path: packages
-
-      - name: Create release json
-        run: |
-          cd packages
-          "$GITHUB_WORKSPACE/tools/generate-release-json" "$BASE_URL" > nightly.json
-        env:
-          BASE_URL: https://syncthing.ams3.digitaloceanspaces.com/nightly/
-
-      - name: Push artifacts
-        uses: docker://docker.io/rclone/rclone:latest
-        env:
-          RCLONE_CONFIG_SPACES_TYPE: s3
-          RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
-          RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
-          RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
-          RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
-          RCLONE_CONFIG_SPACES_ACL: public-read
-        with:
-          args: sync packages spaces:syncthing/nightly
-
-  #
-  # Push release artifacts to Spaces
-  #
-
-  publish-release-files:
-    name: Publish release files
-    if: github.event_name == 'push' && github.ref == 'refs/heads/release'
-    environment: signing
-    needs:
-      - sign-for-upgrade
-      - package-debian
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Download signed packages
-        uses: actions/download-artifact@v3
-        with:
-          name: packages-signed
-          path: packages
-
-      - name: Download debian packages
-        uses: actions/download-artifact@v3
-        with:
-          name: debian-packages
-          path: packages
-
-      - name: Set version
-        run: |
-          version=$(go run build.go version)
-          echo "VERSION=$version" >> $GITHUB_ENV
-
-      - name: Push to Spaces (${{ env.VERSION }})
-        uses: docker://docker.io/rclone/rclone:latest
-        env:
-          RCLONE_CONFIG_SPACES_TYPE: s3
-          RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
-          RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
-          RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
-          RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
-          RCLONE_CONFIG_SPACES_ACL: public-read
-        with:
-          args: sync packages spaces:syncthing/release/${{ env.VERSION }}
-
-      - name: Push to Spaces (latest)
-        uses: docker://docker.io/rclone/rclone:latest
-        env:
-          RCLONE_CONFIG_SPACES_TYPE: s3
-          RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
-          RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
-          RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
-          RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
-          RCLONE_CONFIG_SPACES_ACL: public-read
-        with:
-          args: sync spaces:syncthing/release/${{ env.VERSION }} spaces:syncthing/release/latest
-
-  #
-  # Build and push to Docker Hub
-  #
-
-  docker-syncthing:
-    name: Build and push Docker images
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push' && (github.ref == 'refs/heads/release' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release-'))
-    environment: docker
-    strategy:
-      matrix:
-        pkg:
-          - syncthing
-          - strelaysrv
-          - stdiscosrv
-        include:
-          - pkg: syncthing
-            dockerfile: Dockerfile
-            image: syncthing/syncthing
-          - pkg: strelaysrv
-            dockerfile: Dockerfile.strelaysrv
-            image: syncthing/relaysrv
-          - pkg: stdiscosrv
-            dockerfile: Dockerfile.stdiscosrv
-            image: syncthing/discosrv
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-docker-${{ matrix.pkg }}-${{ hashFiles('**/go.sum') }}
-
-      - name: Build binaries
-        run: |
-          for arch in amd64 arm64 arm; do
-            go run build.go -goos linux -goarch "$arch" -no-upgrade build ${{ matrix.pkg }}
-            mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-"$arch"
-          done
-        env:
-          CGO_ENABLED: "0"
-          BUILD_USER: docker
-
-      - name: Check if we will be able to push images
-        run: |
-          if [[ "${{ secrets.DOCKERHUB_TOKEN }}" != "" ]]; then
-            echo "DOCKER_PUSH=true" >> $GITHUB_ENV;
-          fi
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        if: env.DOCKER_PUSH == 'true'
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Set version tags
-        run: |
-          version=$(go run build.go version)
-          version=${version#v}
-          if [[ $version == @([0-9]|[0-9][0-9]).@([0-9]|[0-9][0-9]).@([0-9]|[0-9][0-9]) ]] ; then
-            echo Release version, pushing to :latest and version tags
-            major=${version%.*.*}
-            minor=${version%.*}
-            tags=${{ matrix.image }}:$version,${{ matrix.image }}:$major,${{ matrix.image }}:$minor,${{ matrix.image }}:latest
-          elif [[ $version == *-rc.@([0-9]|[0-9][0-9]) ]] ; then
-            echo Release candidate, pushing to :rc
-            tags=${{ matrix.image }}:rc
-          else
-            echo Development version, pushing to :edge
-            tags=${{ matrix.image }}:edge
-          fi
-          echo "DOCKER_TAGS=$tags" >> $GITHUB_ENV
-
-      - name: Build and push Docker image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ${{ matrix.dockerfile }}
-          platforms: linux/amd64,linux/arm64,linux/arm/7
-          push: ${{ env.DOCKER_PUSH == 'true' }}
-          tags: ${{ env.DOCKER_TAGS }}
-
-  #
-  # Check for known vulnerabilities in Go dependencies
-  #
-
-  govulncheck:
-    runs-on: ubuntu-latest
-    name: Run govulncheck
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-          check-latest: true
-
-      - name: run govulncheck
-        run: |
-          go run build.go assets
-          go install golang.org/x/vuln/cmd/govulncheck@latest
-          govulncheck ./...

.github/workflows/trigger-nightly.yaml

@@ -1,21 +0,0 @@
-name: Trigger nightly build & release
-on:
-  workflow_dispatch:
-  schedule:
-    # Run nightly build at 01:00 UTC
-    - cron: '00 01 * * *'
-
-jobs:
-
-  trigger-nightly:
-    runs-on: ubuntu-latest
-    name: Push to release-nightly to trigger build
-    steps:
-
-      - uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
-          fetch-depth: 0
-
-      - run: |
-          git push origin main:release-nightly

.github/workflows/update-docs-translations.yaml

@@ -1,28 +0,0 @@
-name: Update translations and documentation
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: '42 3 * * 1'
-
-jobs:
-
-  update_transifex_docs:
-    runs-on: ubuntu-latest
-    name: Update translations and documentation
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ^1.19.6
-      - run: |
-          set -euo pipefail
-          git config --global user.name 'Syncthing Release Automation'
-          git config --global user.email 'release@syncthing.net'
-          bash build.sh translate
-          bash build.sh prerelease
-          git push
-        env:
-          WEBLATE_TOKEN: ${{ secrets.WEBLATE_TOKEN }}

.gitignore

@@ -1,10 +1,11 @@
 /syncthing
 /stdiscosrv
+syncthing.exe
+stdiscosrv.exe
 *.tar.gz
 *.zip
 *.asc
 *.deb
-*.exe
 .jshintrc
 coverage.out
 files/pidx
@@ -14,7 +15,5 @@ coverage.xml
 syncthing.sig
 RELEASE
 deb
-*.bz2
-/repos
-/proto/scripts/protoc-gen-gosyncthing
-/gui/next-gen-gui
+lib/auto/gui.files.go
+snapcraft.yaml

.golangci.yml

@@ -1,26 +0,0 @@
-linters-settings:
-  maligned:
-    suggest-new: true
-
-linters:
-  enable-all: true
-  disable:
-    - goimports
-    - depguard
-    - lll
-    - gochecknoinits
-    - gochecknoglobals
-    - gofmt
-    - scopelint
-    - gocyclo
-    - funlen
-    - wsl
-    - gocognit
-    - godox
-
-service:
-  golangci-lint-version: 1.21.x
-  prepare:
-    - rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
-    - GO111MODULE=on go mod vendor
-    - go run build.go assets

.mailmap

@@ -0,0 +1 @@
+NICKS

.yamlfmt

@@ -1,4 +0,0 @@
-line_ending: lf
-formatter:
-  type: basic
-  retain_line_breaks: true

AUTHORS

@@ -1,324 +1,120 @@
 # This is the official list of Syncthing authors for copyright purposes.
-#
-# THIS FILE IS MOSTLY AUTO GENERATED. IF YOU'VE MADE A COMMIT TO THE
-# REPOSITORY YOU WILL BE ADDED HERE AUTOMATICALLY WITHOUT THE NEED FOR
-# ANY MANUAL ACTION.
-#
-# That said, you are welcome to correct your name or add a nickname / GitHub
-# user name as appropriate. The format is:
+# The format is:
 #
 #    Name Name Name (nickname) <email1@example.com> <email2@example.com>
 #
-# The in-GUI authors list is periodically automatically updated from the
-# contents of this file.
-#
+# The NICKS list is auto generated from this file.
 
 Aaron Bieber (qbit) <qbit@deftly.net>
-Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com> <adam@proactiveservices.co.uk>
+Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com>
 Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
-Alan Pope <alan@popey.com>
-Alberto Donato <albertodonato@users.noreply.github.com>
-Aleksey Vasenev <margtu-fivt@ya.ru>
 Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
-Alex Lindeman <139387+aelindeman@users.noreply.github.com>
-Alex Xu <alex.hello71@gmail.com>
 Alexander Graf (alex2108) <register-github@alex-graf.de>
-Alexander Seiler <seileralex@gmail.com>
-Alexandre Alves <alexandrealvesdb.contact@gmail.com>
 Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
-Aman Gupta <aman@tmm1.net>
 Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
-Andreas Sommer <andreas.sommer87@googlemail.com>
-andresvia <andres.via@gmail.com>
 Andrew Dunham (andrew-d) <andrew@du.nham.ca>
-Andrew Meyer <andrewm.bpi@gmail.com>
-Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
-Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
-André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
-andyleap <andyleap@gmail.com>
-Anjan Momi <anjan@momi.ca>
-Anthony Goeckner <agoeckner@users.noreply.github.com>
+Andrey D (scienmind) <scintertech@cryptolab.net>
 Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
 Antony Male (canton7) <antony.male@gmail.com>
-Anur <anurnomeru@163.com>
-Aranjedeath <Aranjedeath@users.noreply.github.com>
-Arkadiusz Tymiński <gevleeog@gmail.com>
-Aroun <login@b-vo.fr>
 Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
-Artur Zubilewicz <AkaZecik@users.noreply.github.com>
-Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
-Aurélien Rainone <476650+arl@users.noreply.github.com>
-BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
+Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com>
 Bart De Vries (mogwa1) <devriesb@gmail.com>
 Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
 Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
 Ben Shepherd (benshep) <bjashepherd@gmail.com>
 Ben Sidhom (bsidhom) <bsidhom@gmail.com>
-Benedikt Heine (bebehei) <bebe@bebehei.de>
-Benedikt Morbach <benedikt.morbach@googlemail.com>
-Benjamin Nater <17193640+bn4t@users.noreply.github.com>
-Benno Fünfstück <benno.fuenfstueck@gmail.com>
 Benny Ng (tpng) <benny.tpng@gmail.com>
-boomsquared <54829195+boomsquared@users.noreply.github.com>
-Boqin Qin <bobbqqin@bupt.edu.cn>
-Boris Rybalkin <ribalkin@gmail.com>
 Brandon Philips (philips) <brandon@ifup.org>
 Brendan Long (brendanlong) <self@brendanlong.com>
 Brian R. Becker (brbecker) <brbecker@gmail.com>
-bt90 <btom1990@googlemail.com>
 Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
-Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
-Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
+Carsten Hagemann (Moter8) <moter8@gmail.com>
+Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
 Cedric Staniewski (xduugu) <cedric@gmx.ca>
-chenrui <rui@meetup.com>
-Chih-Hsuan Yen <yan12125@gmail.com> <1937689+yan12125@users.noreply.github.com>
-Choongkyu <choongkyu.kim+gh@gmail.com> <vapidlyrapid+gh@gmail.com>
 Chris Howie (cdhowie) <me@chrishowie.com>
 Chris Joel (cdata) <chris@scriptolo.gy>
-Chris Tonkinson <chris@masterbran.ch>
-Christian Kujau <ckujau@users.noreply.github.com>
-Christian Prescott <me@christianprescott.com>
-chucic <chucic@seznam.cz>
 Colin Kennedy (moshen) <moshen.colin@gmail.com>
-Cromefire_ <tim.l@nghorst.net> <26320625+cromefire@users.noreply.github.com>
-cui fliter <imcusg@gmail.com>
-Cyprien Devillez <cypx@users.noreply.github.com>
-Dale Visser <dale.visser@live.com>
-Dan <benda.daniel@gmail.com>
-Daniel Barczyk <46358936+DanielBarczyk@users.noreply.github.com>
 Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
 Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
 Daniel Martí (mvdan) <mvdan@mvdan.cc>
 Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
 David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
-deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com>
 Denis A. (dva) <denisva@gmail.com>
 Dennis Wilson (snnd) <dw@risu.io>
-dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com> <27856297+dependabot-preview[bot]@users.noreply.github.com>
-dependabot[bot] <dependabot[bot]@users.noreply.github.com> <49699333+dependabot[bot]@users.noreply.github.com>
-derekriemer <derek.riemer@colorado.edu>
-desbma <desbma@users.noreply.github.com>
-Devon G. Redekopp <devon@redekopp.com>
-Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com>
-Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
-Domenic Horner <domenic@tgxn.net>
 Dominik Heidler (asdil12) <dominik@heidler.eu>
 Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
-Elliot Huffman <thelich2@gmail.com>
 Emil Hessman (ceh) <emil@hessman.se>
-Emil Lundberg <emil@emlun.se>
-Eng Zer Jun <engzerjun@gmail.com>
-entity0xfe <109791748+entity0xfe@users.noreply.github.com> <entity0xfe@my.domain>
-Eric Lesiuta <elesiuta@gmail.com>
-Eric P <eric@kastelo.net>
 Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
-Evan Spensley <94762716+0evan@users.noreply.github.com>
-Evgeny Kuznetsov <evgeny@kuznetsov.md>
 Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
-Felix <53702818+f-eliks@users.noreply.github.com>
 Felix Ableitner (Nutomic) <me@nutomic.com>
-Felix Lampe <mail@flampe.de>
 Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
 Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
 Frank Isemann (fti7) <frank@isemann.name>
-Gahl Saraf <saraf.gahl@gmail.com> <gahl@raftt.io>
-georgespatton <georgespatton@users.noreply.github.com>
-ghjklw <malo@jaffre.info>
 Gilli Sigurdsson (gillisig) <gilli@vx.is>
-Gleb Sinyavskiy <zhulik.gleb@gmail.com>
-Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
-greatroar <61184462+greatroar@users.noreply.github.com>
-Greg <gco@jazzhaiku.com>
-guangwu <guoguangwu@magic-shield.com>
-Han Boetes <han@boetes.org>
-HansK-p <42314815+HansK-p@users.noreply.github.com>
-Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
 Heiko Zuerker (Smiley73) <heiko@zuerker.org>
-Hugo Locurcio <hugo.locurcio@hugo.pro>
-Iain Barnett <iainspeed@gmail.com>
-Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
-ignacy123 <ignacy.buczek@onet.pl>
-Ikko Ashimine <eltociear@gmail.com>
-Ilya Brin <464157+ilyabrin@users.noreply.github.com>
-Iskander Sharipov (Alex) <quasilyte@gmail.com>
 Jaakko Hannikainen (jgke) <jgke@jgke.fi>
 Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
-Jack Croft <jccroft1@users.noreply.github.com>
-Jacob <jyundt@gmail.com>
 Jake Peterson (acogdev) <jake@acogdev.com>
 Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
-James O'Beirne <wild-github@au92.org>
 James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
-janost <janost@tuta.io>
-Jaroslav Lichtblau <svetlemodry@users.noreply.github.com>
 Jaroslav Malec (dzarda) <dzardacz@gmail.com>
-jaseg <githubaccount@jaseg.net>
-Jauder Ho <jauderho@users.noreply.github.com>
 Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
-Jaya Kumar <jaya.kumar@ict.nl>
-Jeffery To <jeffery.to@gmail.com>
-jelle van der Waa <jelle@vdwaa.nl>
 Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
-Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
-Jesse Lucas <jesse@jesselucas.com>
 Jochen Voss (seehuhn) <voss@seehuhn.de>
-Johan Andersson <j@i19.se>
 Johan Vromans (sciurius) <jvromans@squirrel.nl>
-John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
-Jonas Thelemann <e-mail@jonas-thelemann.de>
-Jonathan <artback@protonmail.com> <jonagn@gmail.com>
-Jonathan Cross <jcross@gmail.com>
-Jonta <359397+Jonta@users.noreply.github.com>
 Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
-jtagcat <git-514635f7@jtag.cat> <git-12dbd862@jtag.cat>
-Jörg Thalheim <Mic92@users.noreply.github.com>
-Jędrzej Kula <kula.jedrek@gmail.com>
-K.B.Dharun Krishna <kbdharunkrishna@gmail.com>
-Kalle Laine <pahakalle@protonmail.com>
 Karol Różycki (krozycki) <rozycki.karol@gmail.com>
-Kebin Liu <lkebin@gmail.com>
-Keith Harrison <keithh@protonmail.com>
-Keith Turner <kturner@apache.org>
 Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
 Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
 Kevin Allen (ironmig) <kma1660@gmail.com>
-Kevin Bushiri (keevBush) <keevbush@gmail.com> <36192217+keevBush@users.noreply.github.com>
 Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
-klemens <ka7@github.com>
 Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
 Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
-Lars Lehtonen <lars.lehtonen@gmail.com>
-Laurent Arnoud <laurent@spkdev.net>
 Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
 Leo Arias (elopio) <yo@elopio.net>
 Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
 Lode Hoste (Zillode) <zillode@zillode.be>
 Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
-LSmithx2 <42276854+lsmithx2@users.noreply.github.com>
-Lukas Lihotzki <lukas@lihotzki.de>
-luzpaz <luzpaz@users.noreply.github.com>
 Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
 Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
 Marc Pujol (kilburn) <kilburn@la3.org>
 Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
-marco-m <marco.molteni@laposte.net>
-Marcus Legendre <marcus.legendre@gmail.com>
-Mario Majila <mariustshipichik@gmail.com>
 Mark Pulford (mpx) <mark@kyne.com.au>
-Martchus <martchus@gmx.net>
 Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
-Mateusz Ż <thedead4fun@live.com>
-Matic Potočnik <hairyfotr@gmail.com>
 Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
-Matt Robenolt <matt@ydekproductions.com>
-Matteo Ruina <matteo.ruina@gmail.com>
-Maurizio Tomasi <ziotom78@gmail.com>
-Max <github@germancoding.com>
 Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
-MaximAL <almaximal@ya.ru>
-Maxime Thirouin <m@moox.io>
-Maximilian <maxi.rostock@outlook.de>
-mclang <1721600+mclang@users.noreply.github.com>
 Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
 Michael Ploujnikov (plouj) <ploujj@gmail.com>
-Michael Rienstra <mrienstra@gmail.com>
 Michael Tilli (pyfisch) <pyfisch@gmail.com>
-MichaIng <micha@dietpi.com>
-Migelo <miha@filetki.si>
-Mike Boone <mike@boonedocks.net>
-MikeLund <MikeLund@users.noreply.github.com>
-MikolajTwarog <43782609+MikolajTwarog@users.noreply.github.com>
-Mingxuan Lin <gdlmx@users.noreply.github.com>
-mv1005 <49659413+mv1005@users.noreply.github.com>
 Nate Morrison (nrm21) <natemorrison@gmail.com>
-Naveen <172697+naveensrinivasan@users.noreply.github.com>
-Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
-Nick Busey <NickBusey@users.noreply.github.com>
-Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
-Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
-Nicolas Perraut <n.perraut@gmail.com>
 Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
-Nils Jakobi (thunderstorm99) <jakobi.nils@gmail.com>
-NinoM4ster <ninom4ster@gmail.com>
-Nitroretro <43112364+Nitroretro@users.noreply.github.com>
-NoLooseEnds <jon.koslung@gmail.com>
-Oliver Freyermuth <o.freyermuth@googlemail.com>
-otbutz <tbutz@optitool.de>
-Otiel <Otiel@users.noreply.github.com>
-overkill <22098433+0verk1ll@users.noreply.github.com>
-Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
-Pablo <pbaeyens31+github@gmail.com>
 Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
-Paul Brit <paulbrit44@gmail.com>
-Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
-Paweł Rozlach <vespian@users.noreply.github.com>
-perewa <cavalcante.ten@gmail.com>
-Peter Badida <KeyWeeUsr@users.noreply.github.com>
-Peter Dave Hello <hsu@peterdavehello.org>
 Peter Hoeg (peterhoeg) <peter@speartail.com>
-Peter Marquardt (wwwutz) <wwwutz@gmail.com> <wwwutz@googlemail.com>
-Phani Rithvij <phanirithvij2000@gmail.com>
-Phil Davis <phil.davis@inf.org>
 Philippe Schommers (filoozoom) <philippe@schommers.be>
 Phill Luby (pluby) <phill.luby@newredo.com>
-Pier Paolo Ramon <ramonpierre@gmail.com>
 Piotr Bejda (piobpl) <piotrb10@gmail.com>
-Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
-Quentin Hibon <qh.public@yahoo.com>
-Rahmi Pruitt <rjpruitt16@gmail.com>
-red_led <red-led@users.noreply.github.com>
-Richard Hartmann <RichiH@users.noreply.github.com>
 Robert Carosi (nov1n) <robert@carosi.nl>
-Roberto Santalla <roobre@users.noreply.github.com>
-Robin Schoonover <robin@cornhooves.org>
 Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
 Ross Smith II (rasa) <ross@smithii.com>
-rubenbe <github-com-00ff86@vandamme.email>
-Ruslan Yevdokymov <38809160+ruslanye@users.noreply.github.com>
-Ryan Qian <i@bitbili.net>
 Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
 Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
 Scott Klupfel (kluppy) <kluppy@going2blue.com>
-sec65 <106604020+sec65@users.noreply.github.com>
 Sergey Mishin (ralder) <ralder@yandex.ru>
-Shaarad Dalvi <60266155+shaaraddalvi@users.noreply.github.com> <shdalv@microsoft.com>
 Simon Frei (imsodin) <freisim93@gmail.com>
-Simon Mwepu <simonmwepu@gmail.com>
-Sly_tom_cat <slytomcat@mail.ru>
 Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
-Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
-Steven Eckhoff <steven.eckhoff.opensource@gmail.com>
+Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
 Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
-Syncthing Automation <automation@syncthing.net>
-Syncthing Release Automation <release@syncthing.net>
-Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
-Thomas Hipp <thomashipp@gmail.com>
 Tim Abell (timabell) <tim@timwise.co.uk>
 Tim Howes (timhowes) <timhowes@berkeley.edu>
-Tobias Klauser <tobias.klauser@gmail.com>
 Tobias Nygren (tnn2) <tnn@nygren.pp.se>
-Tobias Tom (tobiastom) <t.tom@succont.de>
-Tom Jakubowski <tom@crystae.net>
-Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com> <twilczynski@naver.com>
-Tommy Thorn <tommy-github-email@thorn.ws>
+Tomas Cerveny (kozec) <kozec@kozec.com>
 Tully Robinson (tojrobinson) <tully@tojr.org>
 Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
-Tyler Kropp <kropptyler@gmail.com>
 Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
 Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
 Victor Buinsky (buinsky) <vix_booja@tut.by>
-Vik <63919734+ViktorOn@users.noreply.github.com>
 Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
-villekalliomaki <53118179+villekalliomaki@users.noreply.github.com>
-Vladimir Rusinov <vrusinov@google.com> <vladimir.rusinov@gmail.com>
-wangguoliang <liangcszzu@163.com>
-Will Rouesnel <wrouesnel@wrouesnel.com>
 William A. Kennington III (wkennington) <william@wkennington.com>
-wouter bolsterlee <wouter@bolsterl.ee>
 Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
-xarx00 <xarx00@users.noreply.github.com>
 Xavier O. (damajor) <damajor@gmail.com>
-xjtdy888 (xjtdy888) <xjtdy888@163.com>
 Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
-佛跳墙 <daoquan@qq.com>
-落心 <luoxin.ttt@gmail.com>

CONDUCT.md

@@ -1,73 +1,92 @@
-# Contributor Covenant Code of Conduct
+## Conduct
 
-## Our Pledge
+* We are committed to providing a friendly, safe and welcoming
+  environment for all, regardless of gender, sexual orientation,
+  disability, ethnicity, religion, or similar personal characteristic.
 
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to making participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, gender identity and expression, level of experience,
-education, socio-economic status, nationality, personal appearance, race,
-religion, or sexual identity and orientation.
+* On IRC, please avoid using overtly sexual nicknames or other nicknames
+  that might detract from a friendly, safe and welcoming environment for
+  all.
 
-## Our Standards
+* Please be kind and courteous. There's no need to be mean or rude.
 
-Examples of behavior that contributes to creating a positive environment
-include:
+* Respect that people have differences of opinion and that every design
+  or implementation choice carries a trade-off and numerous costs. There
+  is seldom a right answer.
 
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
+* Please keep unstructured critique to a minimum. If you have solid
+  ideas you want to experiment with, make a fork and see how it works.
 
-Examples of unacceptable behavior by participants include:
+* We will exclude you from interaction if you insult, demean or harass
+  anyone. That is not welcome behaviour. We interpret the term
+  "harassment" as including the definition in the <a
+  href="http://citizencodeofconduct.org/">Citizen Code of Conduct</a>;
+  if you have any lack of clarity about what might be included in that
+  concept, please read their definition. In particular, we don't
+  tolerate behavior that excludes people in socially marginalized
+  groups.
 
-* The use of sexualized language or imagery and unwelcome sexual attention or
-  advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
+* Private harassment is also unacceptable. No matter who you are, if you
+  feel you have been or are being harassed or made uncomfortable by a
+  community member, please contact one of the channel ops or any of the
+  Syncthing core team immediately. Whether you're a regular  contributor
+  or a newcomer, we care about making this community a safe place for
+  you and we've got your back.
 
-## Our Responsibilities
+* Likewise any spamming, trolling, flaming, baiting or other
+  attention-stealing behaviour is not welcome.
 
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
+## Moderation
 
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
+These are the policies for upholding our community's standards of
+conduct in our communication channels, most notably in Syncthing-related
+IRC channels and on the web forum.
 
-## Scope
+1. Remarks that violate the Syncthing standards of conduct, including
+   hateful, hurtful, oppressive, or exclusionary remarks, are not
+   allowed. (Cursing is allowed, but never targeting another user, and
+   never in a hateful manner.)
 
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
+2. Remarks that moderators find inappropriate, whether listed in the
+   code of conduct or not, are also not allowed.
 
-## Enforcement
+3. Moderators will first respond to such remarks with a warning.
 
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at security@syncthing.net. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
+4. If the warning is unheeded, the user will be "kicked," i.e., kicked
+   out of the communication channel to cool off.
 
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
+5. If the user comes back and continues to make trouble, they will be
+   banned, i.e., indefinitely excluded.
 
-## Attribution
+6. Moderators may choose at their discretion to un-ban the user if it
+   was a first offense and they offer the offended party a genuine
+   apology.
 
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+7. If a moderator bans someone and you think it was unjustified, please
+   take it up with that moderator, or with a different moderator, **in
+   private**. Complaints about bans in-channel are not allowed.
 
-[homepage]: https://www.contributor-covenant.org
+8. Moderators are held to a higher standard than other community
+   members. If a moderator creates an inappropriate situation, they
+   should expect less leeway than others.
+
+In the Syncthing community we strive to go the extra step to look out
+for each other. Don't just aim to be technically unimpeachable, try to
+be your best self. In particular, avoid flirting with offensive or
+sensitive issues, particularly if they're off-topic; this all too
+often leads to unnecessary fights, hurt feelings, and damaged trust;
+worse, it can drive people away from the community entirely.
+
+And if someone takes issue with something you said or did, resist the
+urge to be defensive. Just stop doing what it was they complained about
+and apologize. Even if you feel you were misinterpreted or unfairly
+accused, chances are good there was something you could've communicated
+better — remember that it's your responsibility to make your fellow
+community members comfortable. Everyone wants to get along and we are
+all here first and foremost because we want to talk about cool
+technology. You will find that people will be eager to assume good
+intent and forgive as long as you earn their trust.
+
+*Adapted from the [Rust Code of Conduct](https://github.com/rust-lang/rust/wiki/Note-development-policy#conduct)*
+
+*Adapted from the [Node.js Policy on Trolling](http://blog.izs.me/post/30036893703/policy-on-trolling)*

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 ## Reporting Bugs
 
-Please file bugs in the [GitHub Issue
+Please file bugs in the [Github Issue
 Tracker](https://github.com/syncthing/syncthing/issues). Include at
 least the following:
 
@@ -24,15 +24,10 @@ too much information will never get you yelled at. :)
 ## Contributing Translations
 
 All translations are done via
-[Weblate](https://hosted.weblate.org/projects/syncthing/). If you wish
-to contribute to a translation, just head over there and sign up.
+[Transifex](https://www.transifex.com/projects/p/syncthing/). If you
+wish to contribute to a translation, just head over there and sign up.
 Before every release, the language resources are updated from the
-latest info on Weblate.
-
-Note that the previously used service at
-[Transifex](https://www.transifex.com/projects/p/syncthing/) is being
-retired and we kindly ask you to sign up on Weblate for continued
-involvement.
+latest info on Transifex.
 
 ## Contributing Code
 

Dockerfile

@@ -1,49 +0,0 @@
-ARG GOVERSION=latest
-
-#
-# Maybe build Syncthing. This is a bit ugly as we can't make an entire
-# section of the Dockerfile conditional, so we end up always pulling the
-# golang image as builder. Then we check if the executable we need already
-# exists (pre-built) otherwise we build it.
-#
-
-FROM golang:$GOVERSION AS builder
-ARG BUILD_USER
-ARG BUILD_HOST
-ARG TARGETARCH
-
-WORKDIR /src
-COPY . .
-
-ENV CGO_ENABLED=0
-RUN if [ ! -f syncthing-linux-$TARGETARCH ] ; then \
-    go run build.go -no-upgrade build syncthing ; \
-    mv syncthing syncthing-linux-$TARGETARCH ; \
-  fi
-
-#
-# The rest of the Dockerfile uses the binary from the builder, prebuilt or
-# not.
-#
-
-FROM alpine
-ARG TARGETARCH
-
-EXPOSE 8384 22000/tcp 22000/udp 21027/udp
-
-VOLUME ["/var/syncthing"]
-
-RUN apk add --no-cache ca-certificates curl libcap su-exec tzdata
-
-COPY --from=builder /src/syncthing-linux-$TARGETARCH /bin/syncthing
-COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
-
-ENV PUID=1000 PGID=1000 HOME=/var/syncthing
-
-HEALTHCHECK --interval=1m --timeout=10s \
-  CMD curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -o --color=never OK || exit 1
-
-ENV STGUIADDRESS=0.0.0.0:8384
-ENV STHOMEDIR=/var/syncthing/config
-RUN chmod 755 /bin/entrypoint.sh
-ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing"]

Dockerfile.builder

@@ -1,9 +0,0 @@
-ARG GOVERSION=latest
-FROM golang:$GOVERSION
-
-# FPM to build Debian packages
-RUN apt-get update && apt-get install -y --no-install-recommends \
-	locales rubygems ruby-dev build-essential git \
-	&& apt-get clean \
-	&& rm -rf /var/lib/apt/lists/* \
-	&& gem install fpm

Dockerfile.stcrashreceiver

@@ -1,8 +0,0 @@
-FROM alpine
-ARG TARGETARCH
-
-EXPOSE 8080
-
-COPY stcrashreceiver-linux-${TARGETARCH} /bin/stcrashreceiver
-
-ENTRYPOINT [ "/bin/stcrashreceiver" ]

Dockerfile.stdiscosrv

@@ -1,34 +0,0 @@
-ARG GOVERSION=latest
-FROM golang:$GOVERSION AS builder
-ARG BUILD_USER
-ARG BUILD_HOST
-ARG TARGETARCH
-
-WORKDIR /src
-COPY . .
-
-ENV CGO_ENABLED=0
-RUN if [ ! -f stdiscosrv-linux-$TARGETARCH ] ; then \
-    go run build.go -no-upgrade build stdiscosrv ; \
-    mv stdiscosrv stdiscosrv-linux-$TARGETARCH ; \
-  fi
-
-FROM alpine
-ARG TARGETARCH
-
-EXPOSE 19200 8443
-
-VOLUME ["/var/stdiscosrv"]
-
-RUN apk add --no-cache ca-certificates su-exec
-
-COPY --from=builder /src/stdiscosrv-linux-$TARGETARCH /bin/stdiscosrv
-COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
-
-ENV PUID=1000 PGID=1000 HOME=/var/stdiscosrv
-
-HEALTHCHECK --interval=1m --timeout=10s \
-  CMD nc -z localhost 8443 || exit 1
-
-WORKDIR /var/stdiscosrv
-ENTRYPOINT ["/bin/entrypoint.sh", "/bin/stdiscosrv"]

Dockerfile.strelaypoolsrv

@@ -1,16 +0,0 @@
-FROM alpine
-ARG TARGETARCH
-
-EXPOSE 8080
-
-RUN apk add --no-cache ca-certificates su-exec curl
-ENV PUID=1000 PGID=1000 MAXMIND_KEY=
-
-RUN mkdir /var/strelaypoolsrv && chown 1000 /var/strelaypoolsrv
-USER 1000
-
-COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv
-COPY script/strelaypoolsrv-entrypoint.sh /bin/entrypoint.sh
-
-WORKDIR /var/strelaypoolsrv
-ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaypoolsrv", "-listen", ":8080"]

Dockerfile.strelaysrv

@@ -1,34 +0,0 @@
-ARG GOVERSION=latest
-FROM golang:$GOVERSION AS builder
-ARG BUILD_USER
-ARG BUILD_HOST
-ARG TARGETARCH
-
-WORKDIR /src
-COPY . .
-
-ENV CGO_ENABLED=0
-RUN if [ ! -f strelaysrv-linux-$TARGETARCH ] ; then \
-    go run build.go -no-upgrade build strelaysrv ; \
-    mv strelaysrv strelaysrv-linux-$TARGETARCH ; \
-  fi
-
-FROM alpine
-ARG TARGETARCH
-
-EXPOSE 22067 22070
-
-VOLUME ["/var/strelaysrv"]
-
-RUN apk add --no-cache ca-certificates su-exec
-
-COPY --from=builder /src/strelaysrv-linux-$TARGETARCH /bin/strelaysrv
-COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
-
-ENV PUID=1000 PGID=1000 HOME=/var/strelaysrv
-
-HEALTHCHECK --interval=1m --timeout=10s \
-  CMD nc -z localhost 22067 || exit 1
-
-WORKDIR /var/strelaysrv
-ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaysrv"]

Dockerfile.stupgrades

@@ -1,8 +0,0 @@
-FROM alpine
-ARG TARGETARCH
-
-EXPOSE 8080
-
-COPY stupgrades-linux-${TARGETARCH} /bin/stupgrades
-
-ENTRYPOINT [ "/bin/stupgrades" ]

GOALS.md

@@ -36,7 +36,7 @@ or modification by unauthorized parties.
 
 Syncthing should be approachable, understandable and inclusive.
 
-> Complex concepts and maths form the base of Syncthing's functionality.
+> Complex concepts and maths form the base of Synchting's functionality.
 > This should nonetheless be abstracted or hidden to a degree where
 > Syncthing is usable by the general public.
 
@@ -57,7 +57,7 @@ latest technology is not always available to any given individual.
 > Computers include desktops, laptops, servers, virtual machines, small
 > general purpose computers such as Raspberry Pis and, *where possible*,
 > tablets and phones. NAS appliances, toasters, cars, firearms, thermostats
-> and so on may include computing capabilities but it is not our goal for
+> and so on may include computing capabitilies but it is not our goal for
 > Syncthing to run smoothly on these devices.
 
 ### 6. For Individuals

ISSUE_TEMPLATE.md

@@ -0,0 +1,28 @@
+Do not report security issues in this bug tracker. Instead, contact
+security@syncthing.net directly - see https://syncthing.net/security.html
+for more information.
+
+If your issue is a support request ("How do I get my devices to connect?"
+or similar), please use the support forum at https://forum.syncthing.net/
+where a large number of helpful people hang out. This issue tracker is for
+reporting bugs or feature requests directly to the developers.
+
+If your issue is a bug report, replace this boilerplate with a description
+of the problem, being sure to include at least:
+
+ - what happened,
+ - what you expected to happen instead, and
+ - any steps to reproduce the problem.
+
+Also fill out the version information below and add log output or
+screenshots as appropriate.
+
+If your issue is a feature request, simply replace this template text in
+its entirety.
+
+### Version Information
+
+Syncthing Version: v0.x.y
+OS Version: Windows 7 / Ubuntu 14.04 / ...
+Browser Version: (if applicable, for GUI issues)
+

NICKS

@@ -0,0 +1,149 @@
+# This file maps email addresses used in commits to nicks used the changelog.
+# It is auto generated from the AUTHORS file by script/authors.go.
+
+0x010C <antoine.lamielle@0x010c.fr>
+0x010C <gh@0x010c.fr>
+acogdev <jake@acogdev.com>
+adelq <aqalieh95@gmail.com>
+adelq <adelq@users.noreply.github.com>
+alessandro.g89 <alessandro.g89@gmail.com>
+alex2108 <register-github@alex-graf.de>
+andersonvom <andersonvom@gmail.com>
+andrew-d <andrew@du.nham.ca>
+asdil12 <dominik@heidler.eu>
+AudriusButkevicius <audrius.butkevicius@gmail.com>
+aviau <alexandre@alexandreviau.net>
+aviau <aviau@debian.org>
+bencurthoys <ben@bencurthoys.com>
+benshep <bjashepherd@gmail.com>
+bigbear2nd <bigbear2nd@gmail.com>
+brbecker <brbecker@gmail.com>
+brendanlong <self@brendanlong.com>
+brgmnn <dan.arne.bergmann@gmail.com>
+brgmnn <brgmnn@users.noreply.github.com>
+bsidhom <bsidhom@gmail.com>
+buinsky <vix_booja@tut.by>
+burkemw3 <mburke@amplify.com>
+burkemw3 <burkemw3@gmail.com>
+calmh <jakob@nym.se>
+calmh <jakob@kastelo.net>
+canton7 <antony.male@gmail.com>
+Cathryne <cathryne.linenweaver@gmail.com>
+Cathryne <Cathryne@users.noreply.github.com>
+cdata <chris@scriptolo.gy>
+cdhowie <me@chrishowie.com>
+ceh <emil@hessman.se>
+cqcallaw <enlightened.despot@gmail.com>
+damajor <damajor@gmail.com>
+dinosore <dinosore@dbrsoftware.co.uk>
+dtchanpura <dtchanpura@gmail.com>
+dtchanpura <dcprime314@gmail.com>
+dva <denisva@gmail.com>
+dzarda <dzardacz@gmail.com>
+eipiminus1 <eipiminusone+github@gmail.com>
+eipiminus1 <eipiminus1@users.noreply.github.com>
+elopio <yo@elopio.net>
+facastagnini <federico.castagnini@gmail.com>
+filoozoom <philippe@schommers.be>
+frioux <frew@afoolishmanifesto.com>
+frioux <frioux@gmail.com>
+fti7 <frank@isemann.name>
+gillisig <gilli@vx.is>
+hadogenes <szafar@linux.pl>
+imsodin <freisim93@gmail.com>
+ironmig <kma1660@gmail.com>
+jarlebring <jarlebring@gmail.com>
+jayachithra <s.k.jayachithra@gmail.com>
+jedie <github.com@jensdiemer.de>
+jedie <git@jensdiemer.de>
+jgke <jgke@jgke.fi>
+jmdaweb <jmdaweb@hotmail.com>
+jmdaweb <jmdaweb@users.noreply.github.com>
+jpjp <jamespatterson@operamail.com>
+jpjp <jpjp@users.noreply.github.com>
+kamadak <kamada@nanohz.org>
+KayoticSully <kayoticsully@gmail.com>
+kc1212 <kc04bc@gmx.com>
+kc1212 <kc1212@users.noreply.github.com>
+kilburn <kilburn@la3.org>
+kluppy <kluppy@going2blue.com>
+kozec <kozec@kozec.com>
+kralo <max.schulze@online.de>
+kralo <kralo@users.noreply.github.com>
+krozycki <rozycki.karol@gmail.com>
+Kudalufi <kurt@va1der.ca>
+Kudalufi <kurt.fitzner@gmail.com>
+kwhite17 <kevinwhite1710@gmail.com>
+letiemble <laurent.etiemble@gmail.com>
+letiemble <laurent.etiemble@monobjc.net>
+liusy182 <liusy182@gmail.com>
+liusy182 <liusy182@hotmail.com>
+lkwg82 <lkwg82@gmx.de>
+LordLandon <lordlandon@gmail.com>
+majedev <majed.alhajry@gmail.com>
+marcindziadus <dziadus.marcin@gmail.com>
+marclaporte <marc@marclaporte.com>
+marclaporte <marc@laporte.name>
+mateon1 <matin1111@wp.pl>
+mogwa1 <devriesb@gmail.com>
+moshen <moshen.colin@gmail.com>
+Moter8 <moter8@gmail.com>
+mpx <mark@kyne.com.au>
+mvdan <mvdan@mvdan.cc>
+Niller303 <nielsproest@hotmail.com>
+Niller303 <seje.niels@hotmail.com>
+norgeous <daniel@harte.me>
+norgeous <daniel@danielharte.co.uk>
+norgeous <norgeous@users.noreply.github.com>
+nov1n <robert@carosi.nl>
+nrm21 <natemorrison@gmail.com>
+Nutomic <me@nutomic.com>
+pascalj <github@pascalj.com>
+pascalj <mail@pascal-jungblut.com>
+peterhoeg <peter@speartail.com>
+philips <brandon@ifup.org>
+piobpl <piotrb10@gmail.com>
+plouj <ploujj@gmail.com>
+pluby <phill.luby@newredo.com>
+ProactiveServices <aD@simplypeachy.co.uk>
+ProactiveServices <simplypeachy@users.noreply.github.com>
+ProactiveServices <ProactiveServices@users.noreply.github.com>
+pyfisch <pyfisch@gmail.com>
+qbit <qbit@deftly.net>
+ralder <ralder@yandex.ru>
+rasa <ross@smithii.com>
+Rewt0r <rewt0r@gmx.com>
+Rewt0r <Rewt0r@users.noreply.github.com>
+rumpelsepp <stefan@sevenbyte.org>
+rumpelsepp <rumpelsepp@sevenbyte.org>
+sacheendra <sacheendra.t@gmail.com>
+scienmind <scintertech@cryptolab.net>
+sciurius <jvromans@squirrel.nl>
+seehuhn <voss@seehuhn.de>
+Smiley73 <heiko@zuerker.org>
+snnd <dw@risu.io>
+snugghash <suhas.gundimeda@gmail.com>
+snugghash <snugghash@gmail.com>
+Stefan-Code <stefan.github@gmail.com>
+Stefan-Code <Stefan.github@gmail.com>
+timabell <tim@timwise.co.uk>
+timhowes <timhowes@berkeley.edu>
+tnn2 <tnn@nygren.pp.se>
+tojrobinson <tully@tojr.org>
+tpng <benny.tpng@gmail.com>
+tylerbrazier <tyler@tylerbrazier.com>
+Unrud <unrud@openaliasbox.org>
+Unrud <Unrud@users.noreply.github.com>
+uok <ueomkail@gmail.com>
+uok <uok@users.noreply.github.com>
+veeti <veeti.paananen@rojekti.fi>
+Vilbrekin <vilbrekin@gmail.com>
+wkennington <william@wkennington.com>
+WSGCSysadmin <e.meitner@willystreet.coop>
+wweich <wweich@users.noreply.github.com>
+wweich <wweich@gmx.de>
+wweich <wulf@weich-kr.de>
+xduugu <cedric@gmx.ca>
+zaynetro <romanznet@gmail.com>
+Zillode <zillode@zillode.be>
+zukoo <fxgsell@gmail.com>

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,32 @@
+### Purpose
+
+Describe the purpose of this change. If there is an existing issue that is
+resolved by this pull request, ensure that the commit subject is on the form
+`Some short description (fixes #1234)` where 1234 is the issue number.
+
+### Testing
+
+Describe what testing has been done, and how the reviewer can test the change
+if new tests are not included.
+
+### Screenshots
+
+If this is a GUI change, include screenshots of the change. If not, please
+feel free to just delete this section.
+
+### Documentation
+
+If this is a user visible change (including API and protocol changes), add a link here
+to the corresponding pull request on https://github.com/syncthing/docs or describe
+the documentation changes necessary.
+
+### Authorship
+
+Every author of a code contribution (Go, Javascript, HTML, CSS etc, with the
+possible exception of minor typo corrections and similar) is recorded in the
+AUTHORS and NICKS files and the in-GUI credits. If this is your first
+contribution, a maintainer will add you properly before accepting the
+contribution. You need not do so yourself or worry about the fact that the
+"authors" automated test fails. However, if your name (such as you want it
+presented in the credits) is not visible on your Github profile or in your
+commit messages, please assist by providing it here.

README-Docker.md

@@ -1,109 +0,0 @@
-# Docker Container for Syncthing
-
-Use the Dockerfile in this repo, or pull the `syncthing/syncthing` image
-from Docker Hub.
-
-Use the `/var/syncthing` volume to have the synchronized files available on the
-host. You can add more folders and map them as you prefer.
-
-Note that Syncthing runs as UID 1000 and GID 1000 by default. These may be
-altered with the `PUID` and `PGID` environment variables. In addition
-the name of the Syncthing instance can be optionally defined by using
-`--hostname=syncthing` parameter.
-
-To grant Syncthing additional capabilities without running as root, use the
-`PCAP` environment variable with the same syntax as that for `setcap(8)`.
-For example, `PCAP=cap_chown,cap_fowner+ep`.
-
-## Example Usage
-
-**Docker cli**
-```
-$ docker pull syncthing/syncthing
-$ docker run -p 8384:8384 -p 22000:22000/tcp -p 22000:22000/udp -p 21027:21027/udp \
-    -v /wherever/st-sync:/var/syncthing \
-    --hostname=my-syncthing \
-    syncthing/syncthing:latest
-```
-
-**Docker compose**
-```yml
----
-version: "3"
-services:
-  syncthing:
-    image: syncthing/syncthing
-    container_name: syncthing
-    hostname: my-syncthing
-    environment:
-      - PUID=1000
-      - PGID=1000
-    volumes:
-      - /wherever/st-sync:/var/syncthing
-    ports:
-      - 8384:8384 # Web UI
-      - 22000:22000/tcp # TCP file transfers
-      - 22000:22000/udp # QUIC file transfers
-      - 21027:21027/udp # Receive local discovery broadcasts
-    restart: unless-stopped
-```
-
-## Discovery
-
-Note that Docker's default network mode prevents local IP addresses from
-being discovered, as Syncthing is only able to see the internal IP of the
-container on the `172.17.0.0/16` subnet. This will result in poor transfer rates
-if local device addresses are not manually configured.
-
-It is therefore advisable to use the [host network mode](https://docs.docker.com/network/host/) instead:
-
-**Docker cli**
-```
-$ docker pull syncthing/syncthing
-$ docker run --network=host \
-    -v /wherever/st-sync:/var/syncthing \
-    syncthing/syncthing:latest
-```
-
-**Docker compose**
-```yml
----
-version: "3"
-services:
-  syncthing:
-    image: syncthing/syncthing
-    container_name: syncthing
-    hostname: my-syncthing
-    environment:
-      - PUID=1000
-      - PGID=1000
-    volumes:
-      - /wherever/st-sync:/var/syncthing
-    network_mode: host
-    restart: unless-stopped
-```
-
-Be aware that syncthing alone is now in control of what interfaces and ports it
-listens on. You can edit the syncthing configuration to change the defaults if
-there are conflicts.
-
-## GUI Security
-
-By default Syncthing inside the Docker image listens on 0.0.0.0:8384 to
-allow GUI connections via the Docker proxy. This is set by the
-`STGUIADDRESS` environment variable in the Dockerfile, as it differs from
-what Syncthing would otherwise use by default. This means you should set up
-authentication in the GUI, like for any other externally reachable Syncthing
-instance. If you do not require the GUI, or you use host networking, you can
-unset the `STGUIADDRESS` variable to have Syncthing fall back to listening
-on 127.0.0.1:
-
-```
-$ docker pull syncthing/syncthing
-$ docker run -e STGUIADDRESS= \
-    -v /wherever/st-sync:/var/syncthing \
-    syncthing/syncthing:latest
-```
-
-With the environment variable unset Syncthing will follow what is set in the
-configuration file / GUI settings dialog.

README.md

@@ -2,6 +2,11 @@
 
 ---
 
+[![Latest Linux & Cross Build](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square&label=linux+%26+cross)](https://build.syncthing.net/job/syncthing/lastBuild/)
+[![Latest Windows Build](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing-windows.svg?style=flat-square&label=windows)](https://build.syncthing.net/job/syncthing/lastBuild/)
+[![Latest Mac Build](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing-mac.svg?style=flat-square&label=mac)](https://build.syncthing.net/job/syncthing/lastBuild/)
+[![Latest Solaris Build](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing-solaris.svg?style=flat-square&label=solaris)](https://build.syncthing.net/job/syncthing/lastBuild/)
+[![API Documentation](https://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](https://godoc.org/github.com/syncthing/syncthing)
 [![MPLv2 License](https://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/88/badge)](https://bestpractices.coreinfrastructure.org/projects/88)
 [![Go Report Card](https://goreportcard.com/badge/github.com/syncthing/syncthing)](https://goreportcard.com/report/github.com/syncthing/syncthing)
@@ -16,36 +21,36 @@ commentary, see the full [Goals document][13].
 
 Syncthing should be:
 
-1. **Safe From Data Loss**
+1. Safe From Data Loss
 
    Protecting the user's data is paramount. We take every reasonable
    precaution to avoid corrupting the user's files.
 
-2. **Secure Against Attackers**
+2. Secure Against Attackers
 
    Again, protecting the user's data is paramount. Regardless of our other
    goals we must never allow the user's data to be susceptible to
    eavesdropping or modification by unauthorized parties.
 
-3. **Easy to Use**
+3. Easy to Use
 
    Syncthing should be approachable, understandable and inclusive.
 
-4. **Automatic**
+4. Automatic
 
    User interaction should be required only when absolutely necessary.
 
-5. **Universally Available**
+5. Universally Available
 
    Syncthing should run on every common computer. We are mindful that the
    latest technology is not always available to any given individual.
 
-6. **For Individuals**
+6. For Individuals
 
    Syncthing is primarily about empowering the individual user with safe,
    secure and easy to use file synchronization.
 
-7. **Everything Else**
+7. Everything Else
 
    There are many things we care about that don't make it on to the list. It
    is fine to optimize for these values, as long as they are not in conflict
@@ -59,10 +64,6 @@ There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory][3]. There are also several [GUI
 implementations][11] for Windows, Mac and Linux.
 
-## Docker
-
-To run Syncthing in Docker, see [the Docker README][16].
-
 ## Vote on features/bugs
 
 We'd like to encourage you to [vote][12] on issues that matter to you.
@@ -70,16 +71,15 @@ This helps the team understand what are the biggest pain points for our users, a
 
 ## Getting in Touch
 
-The first and best point of contact is the [Forum][8].
-If you've found something that is clearly a
+The first and best point of contact is the [Forum][8]. There is also an IRC
+channel, `#syncthing` on [freenode][4] (with a [web client][9]), for talking
+directly to developers and users. If you've found something that is clearly a
 bug, feel free to report it in the [GitHub issue tracker][10].
 
 ## Building
 
-Building Syncthing from source is easy. After extracting the source bundle from
-a release or checking out git, you just need to run `go run build.go` and the
-binaries are created in `./bin`. There's [a guide][5] with more details on the
-build process.
+Building Syncthing from source is easy, and there's a [guide][5]
+that describes it for both Unix and Windows systems.
 
 ## Signed Releases
 
@@ -88,27 +88,27 @@ D26E6ED000654A3E, available from https://syncthing.net/security.html and
 most key servers.
 
 There is also a built in automatic upgrade mechanism (disabled in some
-distribution channels) which uses a compiled in ECDSA signature. macOS
-binaries are also properly code signed.
+distribution channels) which uses a compiled in ECDSA signature. Mac OS
+X binaries are also properly code signed.
 
 ## Documentation
 
-Please see the Syncthing [documentation site][6] [[source]][17].
+Please see the [Syncthing documentation site][6].
 
 All code is licensed under the [MPLv2 License][7].
 
 [1]: https://docs.syncthing.net/specs/bep-v1.html
 [2]: https://docs.syncthing.net/intro/getting-started.html
-[3]: https://github.com/syncthing/syncthing/blob/main/etc
+[3]: https://github.com/syncthing/syncthing/blob/master/etc
+[4]: https://www.freenode.net/
 [5]: https://docs.syncthing.net/dev/building.html
 [6]: https://docs.syncthing.net/
-[7]: https://github.com/syncthing/syncthing/blob/main/LICENSE
+[7]: https://github.com/syncthing/syncthing/blob/master/LICENSE
 [8]: https://forum.syncthing.net/
+[9]: https://kiwiirc.com/client/irc.freenode.net/#syncthing
 [10]: https://github.com/syncthing/syncthing/issues
 [11]: https://docs.syncthing.net/users/contrib.html#gui-wrappers
 [12]: https://www.bountysource.com/teams/syncthing/issues
-[13]: https://github.com/syncthing/syncthing/blob/main/GOALS.md
+[13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
 [14]: assets/logo-text-128.png
-[15]: https://syncthing.net/
-[16]: https://github.com/syncthing/syncthing/blob/main/README-Docker.md
-[17]: https://github.com/syncthing/docs
+[15]: https://syncthing.net/ 

build.ps1

@@ -1,20 +0,0 @@
-function build {
-    go run build.go @args
-}
-
-$cmd, $rest = $args
-switch ($cmd) {
-    "test" {
-        $env:LOGGER_DISCARD=1
-        build test
-    }
-
-    "bench" {
-        $env:LOGGER_DISCARD=1
-        build bench
-    }
-
-    default {
-        build @rest
-    }
-}

build.sh

@@ -2,6 +2,8 @@
 set -euo pipefail
 IFS=$'\n\t'
 
+STTRACE=${STTRACE:-}
+
 script() {
 	name="$1"
 	shift
@@ -13,23 +15,94 @@ build() {
 }
 
 case "${1:-default}" in
+	default)
+		build
+		;;
+
+	clean)
+		build "$@"
+		;;
+
+	tar)
+		build "$@"
+		;;
+
+	assets)
+		build "$@"
+		;;
+
+	xdr)
+		build "$@"
+		;;
+
+	translate)
+		build "$@"
+		;;
+
+	deb)
+		build "$@"
+		;;
+
+	setup)
+		build "$@"
+		;;
+
 	test)
+		ulimit -t 600 &>/dev/null || true
+		ulimit -d 512000 &>/dev/null || true
+		ulimit -m 512000 &>/dev/null || true
 		LOGGER_DISCARD=1 build test
 		;;
 
 	bench)
-		LOGGER_DISCARD=1 build bench
+		LOGGER_DISCARD=1 build bench | script benchfilter
 		;;
 
 	prerelease)
-		script authors
-		build weblate
+		go run script/authors.go
+		build transifex
 		pushd man ; ./refresh.sh ; popd
-		git add -A gui man AUTHORS
-		git commit -m 'gui, man, authors: Update docs, translations, and contributors'
+		git add -A gui man
+		git commit -m 'gui, man: Update docs & translations'
+		;;
+
+	noupgrade)
+		build -no-upgrade tar
+		;;
+
+	all)
+		platforms=(
+			darwin-amd64 dragonfly-amd64 freebsd-amd64 linux-amd64 netbsd-amd64 openbsd-amd64 solaris-amd64 windows-amd64
+			freebsd-386 linux-386 netbsd-386 openbsd-386 windows-386
+			linux-arm linux-arm64 linux-ppc64 linux-ppc64le
+		)
+
+		for plat in "${platforms[@]}"; do
+			echo Building "$plat"
+
+			goos="${plat%-*}"
+			goarch="${plat#*-}"
+			dist="tar"
+
+			if [[ $goos == "windows" ]]; then
+				dist="zip"
+			fi
+
+			build -goos "$goos" -goarch "$goarch" "$dist"
+			echo
+		done
+		;;
+
+	test-xunit)
+		ulimit -t 600 &>/dev/null || true
+		ulimit -d 512000 &>/dev/null || true
+		ulimit -m 512000 &>/dev/null || true
+
+		(GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -v -race ./lib/... ./cmd/... || true) > tests.out
+		go2xunit -output tests.xml -fail < tests.out
 		;;
 
 	*)
-		build "$@"
+		echo "Unknown build command $1"
 		;;
 esac

cmd/stbench/main.go

@@ -0,0 +1,143 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// This doesn't build on Windows due to the Rusage stuff.
+
+// +build !windows
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"runtime"
+	"syscall"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/rc"
+)
+
+var homeDir = "h1"
+var syncthingBin = "./bin/syncthing"
+var test = "scan"
+
+func main() {
+	flag.StringVar(&homeDir, "home", homeDir, "Home directory location")
+	flag.StringVar(&syncthingBin, "bin", syncthingBin, "Binary location")
+	flag.StringVar(&test, "test", test, "Test to run")
+	flag.Parse()
+
+	switch test {
+	case "scan":
+		// scan measures the resource usage required to perform the initial
+		// scan, without cleaning away the database first.
+		testScan()
+	}
+}
+
+// testScan starts a process and reports on the resource usage required to
+// perform the initial scan.
+func testScan() {
+	log.Println("Starting...")
+	p := rc.NewProcess("127.0.0.1:8081")
+	if err := p.Start(syncthingBin, "-home", homeDir, "-no-browser"); err != nil {
+		log.Println(err)
+		return
+	}
+	defer p.Stop()
+
+	wallTime := awaitScanComplete(p)
+
+	report(p, wallTime)
+}
+
+// awaitScanComplete waits for a folder to transition idle->scanning and
+// then scanning->idle and returns the time taken for the scan.
+func awaitScanComplete(p *rc.Process) time.Duration {
+	log.Println("Awaiting scan completion...")
+	var t0, t1 time.Time
+	lastEvent := 0
+loop:
+	for {
+		evs, err := p.Events(lastEvent)
+		if err != nil {
+			continue
+		}
+
+		for _, ev := range evs {
+			if ev.Type == "StateChanged" {
+				data := ev.Data.(map[string]interface{})
+				log.Println(ev)
+
+				if data["to"].(string) == "scanning" {
+					t0 = ev.Time
+					continue
+				}
+
+				if !t0.IsZero() && data["to"].(string) == "idle" {
+					t1 = ev.Time
+					break loop
+				}
+			}
+			lastEvent = ev.ID
+		}
+
+		time.Sleep(250 * time.Millisecond)
+	}
+
+	return t1.Sub(t0)
+}
+
+// report stops the given process and reports on it's resource usage in two
+// ways: human readable to stderr, and CSV to stdout.
+func report(p *rc.Process, wallTime time.Duration) {
+	sv, err := p.SystemVersion()
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	ss, err := p.SystemStatus()
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	proc, err := p.Stop()
+	if err != nil {
+		return
+	}
+
+	rusage, ok := proc.SysUsage().(*syscall.Rusage)
+	if !ok {
+		return
+	}
+
+	log.Println("Version:", sv.Version)
+	log.Println("Alloc:", ss.Alloc/1024, "KiB")
+	log.Println("Sys:", ss.Sys/1024, "KiB")
+	log.Println("Goroutines:", ss.Goroutines)
+	log.Println("Wall time:", wallTime)
+	log.Println("Utime:", time.Duration(rusage.Utime.Nano()))
+	log.Println("Stime:", time.Duration(rusage.Stime.Nano()))
+	if runtime.GOOS == "darwin" {
+		// Darwin reports in bytes, Linux seems to report in KiB even
+		// though the manpage says otherwise.
+		rusage.Maxrss /= 1024
+	}
+	log.Println("MaxRSS:", rusage.Maxrss, "KiB")
+
+	fmt.Printf("%s,%d,%d,%d,%.02f,%.02f,%.02f,%d\n",
+		sv.Version,
+		ss.Alloc/1024,
+		ss.Sys/1024,
+		ss.Goroutines,
+		wallTime.Seconds(),
+		time.Duration(rusage.Utime.Nano()).Seconds(),
+		time.Duration(rusage.Stime.Nano()).Seconds(),
+		rusage.Maxrss)
+}

cmd/stcli/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2014 Audrius Butkevičius
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+- The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cmd/stcli/client.go

@@ -0,0 +1,115 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"net/http"
+	"strings"
+
+	"github.com/AudriusButkevicius/cli"
+)
+
+type APIClient struct {
+	httpClient http.Client
+	endpoint   string
+	apikey     string
+	username   string
+	password   string
+	id         string
+	csrf       string
+}
+
+var instance *APIClient
+
+func getClient(c *cli.Context) *APIClient {
+	if instance != nil {
+		return instance
+	}
+	endpoint := c.GlobalString("endpoint")
+	if !strings.HasPrefix(endpoint, "http") {
+		endpoint = "http://" + endpoint
+	}
+	httpClient := http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: c.GlobalBool("insecure"),
+			},
+		},
+	}
+	client := APIClient{
+		httpClient: httpClient,
+		endpoint:   endpoint,
+		apikey:     c.GlobalString("apikey"),
+		username:   c.GlobalString("username"),
+		password:   c.GlobalString("password"),
+	}
+
+	if client.apikey == "" {
+		request, err := http.NewRequest("GET", client.endpoint, nil)
+		die(err)
+		response := client.handleRequest(request)
+		client.id = response.Header.Get("X-Syncthing-ID")
+		if client.id == "" {
+			die("Failed to get device ID")
+		}
+		for _, item := range response.Cookies() {
+			if item.Name == "CSRF-Token-"+client.id[:5] {
+				client.csrf = item.Value
+				goto csrffound
+			}
+		}
+		die("Failed to get CSRF token")
+	csrffound:
+	}
+	instance = &client
+	return &client
+}
+
+func (client *APIClient) handleRequest(request *http.Request) *http.Response {
+	if client.apikey != "" {
+		request.Header.Set("X-API-Key", client.apikey)
+	}
+	if client.username != "" || client.password != "" {
+		request.SetBasicAuth(client.username, client.password)
+	}
+	if client.csrf != "" {
+		request.Header.Set("X-CSRF-Token-"+client.id[:5], client.csrf)
+	}
+
+	response, err := client.httpClient.Do(request)
+	die(err)
+
+	if response.StatusCode == 404 {
+		die("Invalid endpoint or API call")
+	} else if response.StatusCode == 401 {
+		die("Invalid username or password")
+	} else if response.StatusCode == 403 {
+		if client.apikey == "" {
+			die("Invalid CSRF token")
+		}
+		die("Invalid API key")
+	} else if response.StatusCode != 200 {
+		body := strings.TrimSpace(string(responseToBArray(response)))
+		if body != "" {
+			die(body)
+		}
+		die("Unknown HTTP status returned: " + response.Status)
+	}
+	return response
+}
+
+func httpGet(c *cli.Context, url string) *http.Response {
+	client := getClient(c)
+	request, err := http.NewRequest("GET", client.endpoint+"/rest/"+url, nil)
+	die(err)
+	return client.handleRequest(request)
+}
+
+func httpPost(c *cli.Context, url string, body string) *http.Response {
+	client := getClient(c)
+	request, err := http.NewRequest("POST", client.endpoint+"/rest/"+url, bytes.NewBufferString(body))
+	die(err)
+	return client.handleRequest(request)
+}

cmd/stcli/cmd_devices.go

@@ -0,0 +1,188 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/AudriusButkevicius/cli"
+	"github.com/syncthing/syncthing/lib/config"
+)
+
+func init() {
+	cliCommands = append(cliCommands, cli.Command{
+		Name:     "devices",
+		HideHelp: true,
+		Usage:    "Device command group",
+		Subcommands: []cli.Command{
+			{
+				Name:     "list",
+				Usage:    "List registered devices",
+				Requires: &cli.Requires{},
+				Action:   devicesList,
+			},
+			{
+				Name:     "add",
+				Usage:    "Add a new device",
+				Requires: &cli.Requires{"device id", "device name?"},
+				Action:   devicesAdd,
+			},
+			{
+				Name:     "remove",
+				Usage:    "Remove an existing device",
+				Requires: &cli.Requires{"device id"},
+				Action:   devicesRemove,
+			},
+			{
+				Name:     "get",
+				Usage:    "Get a property of a device",
+				Requires: &cli.Requires{"device id", "property"},
+				Action:   devicesGet,
+			},
+			{
+				Name:     "set",
+				Usage:    "Set a property of a device",
+				Requires: &cli.Requires{"device id", "property", "value..."},
+				Action:   devicesSet,
+			},
+		},
+	})
+}
+
+func devicesList(c *cli.Context) {
+	cfg := getConfig(c)
+	first := true
+	writer := newTableWriter()
+	for _, device := range cfg.Devices {
+		if !first {
+			fmt.Fprintln(writer)
+		}
+		fmt.Fprintln(writer, "ID:\t", device.DeviceID, "\t")
+		fmt.Fprintln(writer, "Name:\t", device.Name, "\t(name)")
+		fmt.Fprintln(writer, "Address:\t", strings.Join(device.Addresses, " "), "\t(address)")
+		fmt.Fprintln(writer, "Compression:\t", device.Compression, "\t(compression)")
+		fmt.Fprintln(writer, "Certificate name:\t", device.CertName, "\t(certname)")
+		fmt.Fprintln(writer, "Introducer:\t", device.Introducer, "\t(introducer)")
+		first = false
+	}
+	writer.Flush()
+}
+
+func devicesAdd(c *cli.Context) {
+	nid := c.Args()[0]
+	id := parseDeviceID(nid)
+
+	newDevice := config.DeviceConfiguration{
+		DeviceID:  id,
+		Name:      nid,
+		Addresses: []string{"dynamic"},
+	}
+
+	if len(c.Args()) > 1 {
+		newDevice.Name = c.Args()[1]
+	}
+
+	if len(c.Args()) > 2 {
+		addresses := c.Args()[2:]
+		for _, item := range addresses {
+			if item == "dynamic" {
+				continue
+			}
+			validAddress(item)
+		}
+		newDevice.Addresses = addresses
+	}
+
+	cfg := getConfig(c)
+	for _, device := range cfg.Devices {
+		if device.DeviceID == id {
+			die("Device " + nid + " already exists")
+		}
+	}
+	cfg.Devices = append(cfg.Devices, newDevice)
+	setConfig(c, cfg)
+}
+
+func devicesRemove(c *cli.Context) {
+	nid := c.Args()[0]
+	id := parseDeviceID(nid)
+	if nid == getMyID(c) {
+		die("Cannot remove yourself")
+	}
+	cfg := getConfig(c)
+	for i, device := range cfg.Devices {
+		if device.DeviceID == id {
+			last := len(cfg.Devices) - 1
+			cfg.Devices[i] = cfg.Devices[last]
+			cfg.Devices = cfg.Devices[:last]
+			setConfig(c, cfg)
+			return
+		}
+	}
+	die("Device " + nid + " not found")
+}
+
+func devicesGet(c *cli.Context) {
+	nid := c.Args()[0]
+	id := parseDeviceID(nid)
+	arg := c.Args()[1]
+	cfg := getConfig(c)
+	for _, device := range cfg.Devices {
+		if device.DeviceID != id {
+			continue
+		}
+		switch strings.ToLower(arg) {
+		case "name":
+			fmt.Println(device.Name)
+		case "address":
+			fmt.Println(strings.Join(device.Addresses, "\n"))
+		case "compression":
+			fmt.Println(device.Compression.String())
+		case "certname":
+			fmt.Println(device.CertName)
+		case "introducer":
+			fmt.Println(device.Introducer)
+		default:
+			die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
+		}
+		return
+	}
+	die("Device " + nid + " not found")
+}
+
+func devicesSet(c *cli.Context) {
+	nid := c.Args()[0]
+	id := parseDeviceID(nid)
+	arg := c.Args()[1]
+	config := getConfig(c)
+	for i, device := range config.Devices {
+		if device.DeviceID != id {
+			continue
+		}
+		switch strings.ToLower(arg) {
+		case "name":
+			config.Devices[i].Name = strings.Join(c.Args()[2:], " ")
+		case "address":
+			for _, item := range c.Args()[2:] {
+				if item == "dynamic" {
+					continue
+				}
+				validAddress(item)
+			}
+			config.Devices[i].Addresses = c.Args()[2:]
+		case "compression":
+			err := config.Devices[i].Compression.UnmarshalText([]byte(c.Args()[2]))
+			die(err)
+		case "certname":
+			config.Devices[i].CertName = strings.Join(c.Args()[2:], " ")
+		case "introducer":
+			config.Devices[i].Introducer = parseBool(c.Args()[2])
+		default:
+			die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
+		}
+		setConfig(c, config)
+		return
+	}
+	die("Device " + nid + " not found")
+}

cmd/stcli/cmd_errors.go

@@ -0,0 +1,67 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/AudriusButkevicius/cli"
+)
+
+func init() {
+	cliCommands = append(cliCommands, cli.Command{
+		Name:     "errors",
+		HideHelp: true,
+		Usage:    "Error command group",
+		Subcommands: []cli.Command{
+			{
+				Name:     "show",
+				Usage:    "Show pending errors",
+				Requires: &cli.Requires{},
+				Action:   errorsShow,
+			},
+			{
+				Name:     "push",
+				Usage:    "Push an error to active clients",
+				Requires: &cli.Requires{"error message..."},
+				Action:   errorsPush,
+			},
+			{
+				Name:     "clear",
+				Usage:    "Clear pending errors",
+				Requires: &cli.Requires{},
+				Action:   wrappedHTTPPost("system/error/clear"),
+			},
+		},
+	})
+}
+
+func errorsShow(c *cli.Context) {
+	response := httpGet(c, "system/error")
+	var data map[string][]map[string]interface{}
+	json.Unmarshal(responseToBArray(response), &data)
+	writer := newTableWriter()
+	for _, item := range data["errors"] {
+		time := item["time"].(string)[:19]
+		time = strings.Replace(time, "T", " ", 1)
+		err := item["error"].(string)
+		err = strings.TrimSpace(err)
+		fmt.Fprintln(writer, time+":\t"+err)
+	}
+	writer.Flush()
+}
+
+func errorsPush(c *cli.Context) {
+	err := strings.Join(c.Args(), " ")
+	response := httpPost(c, "system/error", strings.TrimSpace(err))
+	if response.StatusCode != 200 {
+		err = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
+		body := string(responseToBArray(response))
+		if body != "" {
+			err += "\nBody: " + body
+		}
+		die(err)
+	}
+}

cmd/stcli/cmd_folders.go

@@ -0,0 +1,351 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/AudriusButkevicius/cli"
+	"github.com/syncthing/syncthing/lib/config"
+)
+
+func init() {
+	cliCommands = append(cliCommands, cli.Command{
+		Name:     "folders",
+		HideHelp: true,
+		Usage:    "Folder command group",
+		Subcommands: []cli.Command{
+			{
+				Name:     "list",
+				Usage:    "List available folders",
+				Requires: &cli.Requires{},
+				Action:   foldersList,
+			},
+			{
+				Name:     "add",
+				Usage:    "Add a new folder",
+				Requires: &cli.Requires{"folder id", "directory"},
+				Action:   foldersAdd,
+			},
+			{
+				Name:     "remove",
+				Usage:    "Remove an existing folder",
+				Requires: &cli.Requires{"folder id"},
+				Action:   foldersRemove,
+			},
+			{
+				Name:     "override",
+				Usage:    "Override changes from other nodes for a master folder",
+				Requires: &cli.Requires{"folder id"},
+				Action:   foldersOverride,
+			},
+			{
+				Name:     "get",
+				Usage:    "Get a property of a folder",
+				Requires: &cli.Requires{"folder id", "property"},
+				Action:   foldersGet,
+			},
+			{
+				Name:     "set",
+				Usage:    "Set a property of a folder",
+				Requires: &cli.Requires{"folder id", "property", "value..."},
+				Action:   foldersSet,
+			},
+			{
+				Name:     "unset",
+				Usage:    "Unset a property of a folder",
+				Requires: &cli.Requires{"folder id", "property"},
+				Action:   foldersUnset,
+			},
+			{
+				Name:     "devices",
+				Usage:    "Folder devices command group",
+				HideHelp: true,
+				Subcommands: []cli.Command{
+					{
+						Name:     "list",
+						Usage:    "List of devices which the folder is shared with",
+						Requires: &cli.Requires{"folder id"},
+						Action:   foldersDevicesList,
+					},
+					{
+						Name:     "add",
+						Usage:    "Share a folder with a device",
+						Requires: &cli.Requires{"folder id", "device id"},
+						Action:   foldersDevicesAdd,
+					},
+					{
+						Name:     "remove",
+						Usage:    "Unshare a folder with a device",
+						Requires: &cli.Requires{"folder id", "device id"},
+						Action:   foldersDevicesRemove,
+					},
+					{
+						Name:     "clear",
+						Usage:    "Unshare a folder with all devices",
+						Requires: &cli.Requires{"folder id"},
+						Action:   foldersDevicesClear,
+					},
+				},
+			},
+		},
+	})
+}
+
+func foldersList(c *cli.Context) {
+	cfg := getConfig(c)
+	first := true
+	writer := newTableWriter()
+	for _, folder := range cfg.Folders {
+		if !first {
+			fmt.Fprintln(writer)
+		}
+		fmt.Fprintln(writer, "ID:\t", folder.ID, "\t")
+		fmt.Fprintln(writer, "Path:\t", folder.RawPath, "\t(directory)")
+		fmt.Fprintln(writer, "Folder type:\t", folder.Type, "\t(type)")
+		fmt.Fprintln(writer, "Ignore permissions:\t", folder.IgnorePerms, "\t(permissions)")
+		fmt.Fprintln(writer, "Rescan interval in seconds:\t", folder.RescanIntervalS, "\t(rescan)")
+
+		if folder.Versioning.Type != "" {
+			fmt.Fprintln(writer, "Versioning:\t", folder.Versioning.Type, "\t(versioning)")
+			for key, value := range folder.Versioning.Params {
+				fmt.Fprintf(writer, "Versioning %s:\t %s \t(versioning-%s)\n", key, value, key)
+			}
+		}
+		first = false
+	}
+	writer.Flush()
+}
+
+func foldersAdd(c *cli.Context) {
+	cfg := getConfig(c)
+	abs, err := filepath.Abs(c.Args()[1])
+	die(err)
+	folder := config.FolderConfiguration{
+		ID:      c.Args()[0],
+		RawPath: filepath.Clean(abs),
+	}
+	cfg.Folders = append(cfg.Folders, folder)
+	setConfig(c, cfg)
+}
+
+func foldersRemove(c *cli.Context) {
+	cfg := getConfig(c)
+	rid := c.Args()[0]
+	for i, folder := range cfg.Folders {
+		if folder.ID == rid {
+			last := len(cfg.Folders) - 1
+			cfg.Folders[i] = cfg.Folders[last]
+			cfg.Folders = cfg.Folders[:last]
+			setConfig(c, cfg)
+			return
+		}
+	}
+	die("Folder " + rid + " not found")
+}
+
+func foldersOverride(c *cli.Context) {
+	cfg := getConfig(c)
+	rid := c.Args()[0]
+	for _, folder := range cfg.Folders {
+		if folder.ID == rid && folder.Type == config.FolderTypeSendOnly {
+			response := httpPost(c, "db/override", "")
+			if response.StatusCode != 200 {
+				err := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
+				body := string(responseToBArray(response))
+				if body != "" {
+					err += "\nBody: " + body
+				}
+				die(err)
+			}
+			return
+		}
+	}
+	die("Folder " + rid + " not found or folder not master")
+}
+
+func foldersGet(c *cli.Context) {
+	cfg := getConfig(c)
+	rid := c.Args()[0]
+	arg := strings.ToLower(c.Args()[1])
+	for _, folder := range cfg.Folders {
+		if folder.ID != rid {
+			continue
+		}
+		if strings.HasPrefix(arg, "versioning-") {
+			arg = arg[11:]
+			value, ok := folder.Versioning.Params[arg]
+			if ok {
+				fmt.Println(value)
+				return
+			}
+			die("Versioning property " + c.Args()[1][11:] + " not found")
+		}
+		switch arg {
+		case "directory":
+			fmt.Println(folder.RawPath)
+		case "type":
+			fmt.Println(folder.Type)
+		case "permissions":
+			fmt.Println(folder.IgnorePerms)
+		case "rescan":
+			fmt.Println(folder.RescanIntervalS)
+		case "versioning":
+			if folder.Versioning.Type != "" {
+				fmt.Println(folder.Versioning.Type)
+			}
+		default:
+			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, type, permissions, versioning, versioning-<key>")
+		}
+		return
+	}
+	die("Folder " + rid + " not found")
+}
+
+func foldersSet(c *cli.Context) {
+	rid := c.Args()[0]
+	arg := strings.ToLower(c.Args()[1])
+	val := strings.Join(c.Args()[2:], " ")
+	cfg := getConfig(c)
+	for i, folder := range cfg.Folders {
+		if folder.ID != rid {
+			continue
+		}
+		if strings.HasPrefix(arg, "versioning-") {
+			cfg.Folders[i].Versioning.Params[arg[11:]] = val
+			setConfig(c, cfg)
+			return
+		}
+		switch arg {
+		case "directory":
+			cfg.Folders[i].RawPath = val
+		case "type":
+			var t config.FolderType
+			if err := t.UnmarshalText([]byte(val)); err != nil {
+				die("Invalid folder type: " + err.Error())
+			}
+			cfg.Folders[i].Type = t
+		case "permissions":
+			cfg.Folders[i].IgnorePerms = parseBool(val)
+		case "rescan":
+			cfg.Folders[i].RescanIntervalS = parseInt(val)
+		case "versioning":
+			cfg.Folders[i].Versioning.Type = val
+		default:
+			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, master, permissions, versioning, versioning-<key>")
+		}
+		setConfig(c, cfg)
+		return
+	}
+	die("Folder " + rid + " not found")
+}
+
+func foldersUnset(c *cli.Context) {
+	rid := c.Args()[0]
+	arg := strings.ToLower(c.Args()[1])
+	cfg := getConfig(c)
+	for i, folder := range cfg.Folders {
+		if folder.ID != rid {
+			continue
+		}
+		if strings.HasPrefix(arg, "versioning-") {
+			arg = arg[11:]
+			if _, ok := folder.Versioning.Params[arg]; ok {
+				delete(cfg.Folders[i].Versioning.Params, arg)
+				setConfig(c, cfg)
+				return
+			}
+			die("Versioning property " + c.Args()[1][11:] + " not found")
+		}
+		switch arg {
+		case "versioning":
+			cfg.Folders[i].Versioning.Type = ""
+			cfg.Folders[i].Versioning.Params = make(map[string]string)
+		default:
+			die("Invalid property: " + c.Args()[1] + "\nAvailable properties: versioning, versioning-<key>")
+		}
+		setConfig(c, cfg)
+		return
+	}
+	die("Folder " + rid + " not found")
+}
+
+func foldersDevicesList(c *cli.Context) {
+	rid := c.Args()[0]
+	cfg := getConfig(c)
+	for _, folder := range cfg.Folders {
+		if folder.ID != rid {
+			continue
+		}
+		for _, device := range folder.Devices {
+			fmt.Println(device.DeviceID)
+		}
+		return
+	}
+	die("Folder " + rid + " not found")
+}
+
+func foldersDevicesAdd(c *cli.Context) {
+	rid := c.Args()[0]
+	nid := parseDeviceID(c.Args()[1])
+	cfg := getConfig(c)
+	for i, folder := range cfg.Folders {
+		if folder.ID != rid {
+			continue
+		}
+		for _, device := range folder.Devices {
+			if device.DeviceID == nid {
+				die("Device " + c.Args()[1] + " is already part of this folder")
+			}
+		}
+		for _, device := range cfg.Devices {
+			if device.DeviceID == nid {
+				cfg.Folders[i].Devices = append(folder.Devices, config.FolderDeviceConfiguration{
+					DeviceID: device.DeviceID,
+				})
+				setConfig(c, cfg)
+				return
+			}
+		}
+		die("Device " + c.Args()[1] + " not found in device list")
+	}
+	die("Folder " + rid + " not found")
+}
+
+func foldersDevicesRemove(c *cli.Context) {
+	rid := c.Args()[0]
+	nid := parseDeviceID(c.Args()[1])
+	cfg := getConfig(c)
+	for ri, folder := range cfg.Folders {
+		if folder.ID != rid {
+			continue
+		}
+		for ni, device := range folder.Devices {
+			if device.DeviceID == nid {
+				last := len(folder.Devices) - 1
+				cfg.Folders[ri].Devices[ni] = folder.Devices[last]
+				cfg.Folders[ri].Devices = cfg.Folders[ri].Devices[:last]
+				setConfig(c, cfg)
+				return
+			}
+		}
+		die("Device " + c.Args()[1] + " not found")
+	}
+	die("Folder " + rid + " not found")
+}
+
+func foldersDevicesClear(c *cli.Context) {
+	rid := c.Args()[0]
+	cfg := getConfig(c)
+	for i, folder := range cfg.Folders {
+		if folder.ID != rid {
+			continue
+		}
+		cfg.Folders[i].Devices = []config.FolderDeviceConfiguration{}
+		setConfig(c, cfg)
+		return
+	}
+	die("Folder " + rid + " not found")
+}

cmd/stcli/cmd_general.go

@@ -0,0 +1,78 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/AudriusButkevicius/cli"
+)
+
+func init() {
+	cliCommands = append(cliCommands, []cli.Command{
+		{
+			Name:     "id",
+			Usage:    "Get ID of the Syncthing client",
+			Requires: &cli.Requires{},
+			Action:   generalID,
+		},
+		{
+			Name:     "status",
+			Usage:    "Configuration status, whether or not a restart is required for changes to take effect",
+			Requires: &cli.Requires{},
+			Action:   generalStatus,
+		},
+		{
+			Name:     "restart",
+			Usage:    "Restart syncthing",
+			Requires: &cli.Requires{},
+			Action:   wrappedHTTPPost("system/restart"),
+		},
+		{
+			Name:     "shutdown",
+			Usage:    "Shutdown syncthing",
+			Requires: &cli.Requires{},
+			Action:   wrappedHTTPPost("system/shutdown"),
+		},
+		{
+			Name:     "reset",
+			Usage:    "Reset syncthing deleting all folders and devices",
+			Requires: &cli.Requires{},
+			Action:   wrappedHTTPPost("system/reset"),
+		},
+		{
+			Name:     "upgrade",
+			Usage:    "Upgrade syncthing (if a newer version is available)",
+			Requires: &cli.Requires{},
+			Action:   wrappedHTTPPost("system/upgrade"),
+		},
+		{
+			Name:     "version",
+			Usage:    "Syncthing client version",
+			Requires: &cli.Requires{},
+			Action:   generalVersion,
+		},
+	}...)
+}
+
+func generalID(c *cli.Context) {
+	fmt.Println(getMyID(c))
+}
+
+func generalStatus(c *cli.Context) {
+	response := httpGet(c, "system/config/insync")
+	var status struct{ ConfigInSync bool }
+	json.Unmarshal(responseToBArray(response), &status)
+	if !status.ConfigInSync {
+		die("Config out of sync")
+	}
+	fmt.Println("Config in sync")
+}
+
+func generalVersion(c *cli.Context) {
+	response := httpGet(c, "system/version")
+	version := make(map[string]interface{})
+	json.Unmarshal(responseToBArray(response), &version)
+	prettyPrintJSON(version)
+}

cmd/stcli/cmd_gui.go

@@ -0,0 +1,127 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/AudriusButkevicius/cli"
+)
+
+func init() {
+	cliCommands = append(cliCommands, cli.Command{
+		Name:     "gui",
+		HideHelp: true,
+		Usage:    "GUI command group",
+		Subcommands: []cli.Command{
+			{
+				Name:     "dump",
+				Usage:    "Show all GUI configuration settings",
+				Requires: &cli.Requires{},
+				Action:   guiDump,
+			},
+			{
+				Name:     "get",
+				Usage:    "Get a GUI configuration setting",
+				Requires: &cli.Requires{"setting"},
+				Action:   guiGet,
+			},
+			{
+				Name:     "set",
+				Usage:    "Set a GUI configuration setting",
+				Requires: &cli.Requires{"setting", "value"},
+				Action:   guiSet,
+			},
+			{
+				Name:     "unset",
+				Usage:    "Unset a GUI configuration setting",
+				Requires: &cli.Requires{"setting"},
+				Action:   guiUnset,
+			},
+		},
+	})
+}
+
+func guiDump(c *cli.Context) {
+	cfg := getConfig(c).GUI
+	writer := newTableWriter()
+	fmt.Fprintln(writer, "Enabled:\t", cfg.Enabled, "\t(enabled)")
+	fmt.Fprintln(writer, "Use HTTPS:\t", cfg.UseTLS(), "\t(tls)")
+	fmt.Fprintln(writer, "Listen Addresses:\t", cfg.Address(), "\t(address)")
+	if cfg.User != "" {
+		fmt.Fprintln(writer, "Authentication User:\t", cfg.User, "\t(username)")
+		fmt.Fprintln(writer, "Authentication Password:\t", cfg.Password, "\t(password)")
+	}
+	if cfg.APIKey != "" {
+		fmt.Fprintln(writer, "API Key:\t", cfg.APIKey, "\t(apikey)")
+	}
+	writer.Flush()
+}
+
+func guiGet(c *cli.Context) {
+	cfg := getConfig(c).GUI
+	arg := c.Args()[0]
+	switch strings.ToLower(arg) {
+	case "enabled":
+		fmt.Println(cfg.Enabled)
+	case "tls":
+		fmt.Println(cfg.UseTLS())
+	case "address":
+		fmt.Println(cfg.Address())
+	case "user":
+		if cfg.User != "" {
+			fmt.Println(cfg.User)
+		}
+	case "password":
+		if cfg.User != "" {
+			fmt.Println(cfg.Password)
+		}
+	case "apikey":
+		if cfg.APIKey != "" {
+			fmt.Println(cfg.APIKey)
+		}
+	default:
+		die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
+	}
+}
+
+func guiSet(c *cli.Context) {
+	cfg := getConfig(c)
+	arg := c.Args()[0]
+	val := c.Args()[1]
+	switch strings.ToLower(arg) {
+	case "enabled":
+		cfg.GUI.Enabled = parseBool(val)
+	case "tls":
+		cfg.GUI.RawUseTLS = parseBool(val)
+	case "address":
+		validAddress(val)
+		cfg.GUI.RawAddress = val
+	case "user":
+		cfg.GUI.User = val
+	case "password":
+		cfg.GUI.Password = val
+	case "apikey":
+		cfg.GUI.APIKey = val
+	default:
+		die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
+	}
+	setConfig(c, cfg)
+}
+
+func guiUnset(c *cli.Context) {
+	cfg := getConfig(c)
+	arg := c.Args()[0]
+	switch strings.ToLower(arg) {
+	case "user":
+		cfg.GUI.User = ""
+	case "password":
+		cfg.GUI.Password = ""
+	case "apikey":
+		cfg.GUI.APIKey = ""
+	default:
+		die("Invalid setting: " + arg + "\nAvailable settings: user, password, apikey")
+	}
+	setConfig(c, cfg)
+}

cmd/stcli/cmd_options.go

@@ -0,0 +1,173 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/AudriusButkevicius/cli"
+)
+
+func init() {
+	cliCommands = append(cliCommands, cli.Command{
+		Name:     "options",
+		HideHelp: true,
+		Usage:    "Options command group",
+		Subcommands: []cli.Command{
+			{
+				Name:     "dump",
+				Usage:    "Show all Syncthing option settings",
+				Requires: &cli.Requires{},
+				Action:   optionsDump,
+			},
+			{
+				Name:     "get",
+				Usage:    "Get a Syncthing option setting",
+				Requires: &cli.Requires{"setting"},
+				Action:   optionsGet,
+			},
+			{
+				Name:     "set",
+				Usage:    "Set a Syncthing option setting",
+				Requires: &cli.Requires{"setting", "value..."},
+				Action:   optionsSet,
+			},
+		},
+	})
+}
+
+func optionsDump(c *cli.Context) {
+	cfg := getConfig(c).Options
+	writer := newTableWriter()
+
+	fmt.Fprintln(writer, "Sync protocol listen addresses:\t", strings.Join(cfg.ListenAddresses, " "), "\t(addresses)")
+	fmt.Fprintln(writer, "Global discovery enabled:\t", cfg.GlobalAnnEnabled, "\t(globalannenabled)")
+	fmt.Fprintln(writer, "Global discovery servers:\t", strings.Join(cfg.GlobalAnnServers, " "), "\t(globalannserver)")
+
+	fmt.Fprintln(writer, "Local discovery enabled:\t", cfg.LocalAnnEnabled, "\t(localannenabled)")
+	fmt.Fprintln(writer, "Local discovery port:\t", cfg.LocalAnnPort, "\t(localannport)")
+
+	fmt.Fprintln(writer, "Outgoing rate limit in KiB/s:\t", cfg.MaxSendKbps, "\t(maxsend)")
+	fmt.Fprintln(writer, "Incoming rate limit in KiB/s:\t", cfg.MaxRecvKbps, "\t(maxrecv)")
+	fmt.Fprintln(writer, "Reconnect interval in seconds:\t", cfg.ReconnectIntervalS, "\t(reconnect)")
+	fmt.Fprintln(writer, "Start browser:\t", cfg.StartBrowser, "\t(browser)")
+	fmt.Fprintln(writer, "Enable UPnP:\t", cfg.NATEnabled, "\t(nat)")
+	fmt.Fprintln(writer, "UPnP Lease in minutes:\t", cfg.NATLeaseM, "\t(natlease)")
+	fmt.Fprintln(writer, "UPnP Renewal period in minutes:\t", cfg.NATRenewalM, "\t(natrenew)")
+	fmt.Fprintln(writer, "Restart on Wake Up:\t", cfg.RestartOnWakeup, "\t(wake)")
+
+	reporting := "unrecognized value"
+	switch cfg.URAccepted {
+	case -1:
+		reporting = "false"
+	case 0:
+		reporting = "undecided/false"
+	case 1:
+		reporting = "true"
+	}
+	fmt.Fprintln(writer, "Anonymous usage reporting:\t", reporting, "\t(reporting)")
+
+	writer.Flush()
+}
+
+func optionsGet(c *cli.Context) {
+	cfg := getConfig(c).Options
+	arg := c.Args()[0]
+	switch strings.ToLower(arg) {
+	case "address":
+		fmt.Println(strings.Join(cfg.ListenAddresses, "\n"))
+	case "globalannenabled":
+		fmt.Println(cfg.GlobalAnnEnabled)
+	case "globalannservers":
+		fmt.Println(strings.Join(cfg.GlobalAnnServers, "\n"))
+	case "localannenabled":
+		fmt.Println(cfg.LocalAnnEnabled)
+	case "localannport":
+		fmt.Println(cfg.LocalAnnPort)
+	case "maxsend":
+		fmt.Println(cfg.MaxSendKbps)
+	case "maxrecv":
+		fmt.Println(cfg.MaxRecvKbps)
+	case "reconnect":
+		fmt.Println(cfg.ReconnectIntervalS)
+	case "browser":
+		fmt.Println(cfg.StartBrowser)
+	case "nat":
+		fmt.Println(cfg.NATEnabled)
+	case "natlease":
+		fmt.Println(cfg.NATLeaseM)
+	case "natrenew":
+		fmt.Println(cfg.NATRenewalM)
+	case "reporting":
+		switch cfg.URAccepted {
+		case -1:
+			fmt.Println("false")
+		case 0:
+			fmt.Println("undecided/false")
+		case 1:
+			fmt.Println("true")
+		default:
+			fmt.Println("unknown")
+		}
+	case "wake":
+		fmt.Println(cfg.RestartOnWakeup)
+	default:
+		die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
+	}
+}
+
+func optionsSet(c *cli.Context) {
+	config := getConfig(c)
+	arg := c.Args()[0]
+	val := c.Args()[1]
+	switch strings.ToLower(arg) {
+	case "address":
+		for _, item := range c.Args().Tail() {
+			validAddress(item)
+		}
+		config.Options.ListenAddresses = c.Args().Tail()
+	case "globalannenabled":
+		config.Options.GlobalAnnEnabled = parseBool(val)
+	case "globalannserver":
+		for _, item := range c.Args().Tail() {
+			validAddress(item)
+		}
+		config.Options.GlobalAnnServers = c.Args().Tail()
+	case "localannenabled":
+		config.Options.LocalAnnEnabled = parseBool(val)
+	case "localannport":
+		config.Options.LocalAnnPort = parsePort(val)
+	case "maxsend":
+		config.Options.MaxSendKbps = parseUint(val)
+	case "maxrecv":
+		config.Options.MaxRecvKbps = parseUint(val)
+	case "reconnect":
+		config.Options.ReconnectIntervalS = parseUint(val)
+	case "browser":
+		config.Options.StartBrowser = parseBool(val)
+	case "nat":
+		config.Options.NATEnabled = parseBool(val)
+	case "natlease":
+		config.Options.NATLeaseM = parseUint(val)
+	case "natrenew":
+		config.Options.NATRenewalM = parseUint(val)
+	case "reporting":
+		switch strings.ToLower(val) {
+		case "u", "undecided", "unset":
+			config.Options.URAccepted = 0
+		default:
+			boolvalue := parseBool(val)
+			if boolvalue {
+				config.Options.URAccepted = 1
+			} else {
+				config.Options.URAccepted = -1
+			}
+		}
+	case "wake":
+		config.Options.RestartOnWakeup = parseBool(val)
+	default:
+		die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
+	}
+	setConfig(c, config)
+}

cmd/stcli/cmd_report.go

@@ -0,0 +1,72 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/AudriusButkevicius/cli"
+)
+
+func init() {
+	cliCommands = append(cliCommands, cli.Command{
+		Name:     "report",
+		HideHelp: true,
+		Usage:    "Reporting command group",
+		Subcommands: []cli.Command{
+			{
+				Name:     "system",
+				Usage:    "Report system state",
+				Requires: &cli.Requires{},
+				Action:   reportSystem,
+			},
+			{
+				Name:     "connections",
+				Usage:    "Report about connections to other devices",
+				Requires: &cli.Requires{},
+				Action:   reportConnections,
+			},
+			{
+				Name:     "usage",
+				Usage:    "Usage report",
+				Requires: &cli.Requires{},
+				Action:   reportUsage,
+			},
+		},
+	})
+}
+
+func reportSystem(c *cli.Context) {
+	response := httpGet(c, "system/status")
+	data := make(map[string]interface{})
+	json.Unmarshal(responseToBArray(response), &data)
+	prettyPrintJSON(data)
+}
+
+func reportConnections(c *cli.Context) {
+	response := httpGet(c, "system/connections")
+	data := make(map[string]map[string]interface{})
+	json.Unmarshal(responseToBArray(response), &data)
+	var overall map[string]interface{}
+	for key, value := range data {
+		if key == "total" {
+			overall = value
+			continue
+		}
+		value["Device ID"] = key
+		prettyPrintJSON(value)
+		fmt.Println()
+	}
+	if overall != nil {
+		fmt.Println("=== Overall statistics ===")
+		prettyPrintJSON(overall)
+	}
+}
+
+func reportUsage(c *cli.Context) {
+	response := httpGet(c, "svc/report")
+	report := make(map[string]interface{})
+	json.Unmarshal(responseToBArray(response), &report)
+	prettyPrintJSON(report)
+}

cmd/stcli/labels.go

@@ -0,0 +1,31 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+var jsonAttributeLabels = map[string]string{
+	"folderMaxMiB":   "Largest folder size in MiB",
+	"folderMaxFiles": "Largest folder file count",
+	"longVersion":    "Long version",
+	"totMiB":         "Total size in MiB",
+	"totFiles":       "Total files",
+	"uniqueID":       "Unique ID",
+	"numFolders":     "Folder count",
+	"numDevices":     "Device count",
+	"memoryUsageMiB": "Memory usage in MiB",
+	"memorySize":     "Total memory in MiB",
+	"sha256Perf":     "SHA256 Benchmark",
+	"At":             "Last contacted",
+	"Completion":     "Percent complete",
+	"InBytesTotal":   "Total bytes received",
+	"OutBytesTotal":  "Total bytes sent",
+	"ClientVersion":  "Client version",
+	"alloc":          "Memory allocated in bytes",
+	"sys":            "Memory using in bytes",
+	"cpuPercent":     "CPU load in percent",
+	"extAnnounceOK":  "External announcments working",
+	"goroutines":     "Number of Go routines",
+	"myID":           "Client ID",
+	"tilde":          "Tilde expands to",
+	"arch":           "Architecture",
+	"os":             "OS",
+}

cmd/stcli/main.go

@@ -0,0 +1,63 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"sort"
+
+	"github.com/AudriusButkevicius/cli"
+)
+
+type ByAlphabet []cli.Command
+
+func (a ByAlphabet) Len() int           { return len(a) }
+func (a ByAlphabet) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a ByAlphabet) Less(i, j int) bool { return a[i].Name < a[j].Name }
+
+var cliCommands []cli.Command
+
+func main() {
+	app := cli.NewApp()
+	app.Name = "syncthing-cli"
+	app.Author = "Audrius Butkevičius"
+	app.Email = "audrius.butkevicius@gmail.com"
+	app.Usage = "Syncthing command line interface"
+	app.Version = "0.1"
+	app.HideHelp = true
+
+	app.Flags = []cli.Flag{
+		cli.StringFlag{
+			Name:   "endpoint, e",
+			Value:  "http://127.0.0.1:8384",
+			Usage:  "End point to connect to",
+			EnvVar: "STENDPOINT",
+		},
+		cli.StringFlag{
+			Name:   "apikey, k",
+			Value:  "",
+			Usage:  "API Key",
+			EnvVar: "STAPIKEY",
+		},
+		cli.StringFlag{
+			Name:   "username, u",
+			Value:  "",
+			Usage:  "Username",
+			EnvVar: "STUSERNAME",
+		},
+		cli.StringFlag{
+			Name:   "password, p",
+			Value:  "",
+			Usage:  "Password",
+			EnvVar: "STPASSWORD",
+		},
+		cli.BoolFlag{
+			Name:   "insecure, i",
+			Usage:  "Do not verify SSL certificate",
+			EnvVar: "STINSECURE",
+		},
+	}
+
+	sort.Sort(ByAlphabet(cliCommands))
+	app.Commands = cliCommands
+	app.RunAndExitOnError()
+}

cmd/stcli/utils.go

@@ -0,0 +1,165 @@
+// Copyright (C) 2014 Audrius Butkevičius
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+	"unicode"
+
+	"github.com/AudriusButkevicius/cli"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func responseToBArray(response *http.Response) []byte {
+	defer response.Body.Close()
+	bytes, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		die(err)
+	}
+	return bytes
+}
+
+func die(vals ...interface{}) {
+	if len(vals) > 1 || vals[0] != nil {
+		os.Stderr.WriteString(fmt.Sprintln(vals...))
+		os.Exit(1)
+	}
+}
+
+func wrappedHTTPPost(url string) func(c *cli.Context) {
+	return func(c *cli.Context) {
+		httpPost(c, url, "")
+	}
+}
+
+func prettyPrintJSON(json map[string]interface{}) {
+	writer := newTableWriter()
+	remap := make(map[string]interface{})
+	for k, v := range json {
+		key, ok := jsonAttributeLabels[k]
+		if !ok {
+			key = firstUpper(k)
+		}
+		remap[key] = v
+	}
+
+	jsonKeys := make([]string, 0, len(remap))
+	for key := range remap {
+		jsonKeys = append(jsonKeys, key)
+	}
+	sort.Strings(jsonKeys)
+	for _, k := range jsonKeys {
+		var value string
+		rvalue := remap[k]
+		switch rvalue.(type) {
+		case int, int16, int32, int64, uint, uint16, uint32, uint64, float32, float64:
+			value = fmt.Sprintf("%.0f", rvalue)
+		default:
+			value = fmt.Sprint(rvalue)
+		}
+		if value == "" {
+			continue
+		}
+		fmt.Fprintln(writer, k+":\t"+value)
+	}
+	writer.Flush()
+}
+
+func firstUpper(str string) string {
+	for i, v := range str {
+		return string(unicode.ToUpper(v)) + str[i+1:]
+	}
+	return ""
+}
+
+func newTableWriter() *tabwriter.Writer {
+	writer := new(tabwriter.Writer)
+	writer.Init(os.Stdout, 0, 8, 0, '\t', 0)
+	return writer
+}
+
+func getMyID(c *cli.Context) string {
+	response := httpGet(c, "system/status")
+	data := make(map[string]interface{})
+	json.Unmarshal(responseToBArray(response), &data)
+	return data["myID"].(string)
+}
+
+func getConfig(c *cli.Context) config.Configuration {
+	response := httpGet(c, "system/config")
+	config := config.Configuration{}
+	json.Unmarshal(responseToBArray(response), &config)
+	return config
+}
+
+func setConfig(c *cli.Context, cfg config.Configuration) {
+	body, err := json.Marshal(cfg)
+	die(err)
+	response := httpPost(c, "system/config", string(body))
+	if response.StatusCode != 200 {
+		die("Unexpected status code", response.StatusCode)
+	}
+}
+
+func parseBool(input string) bool {
+	val, err := strconv.ParseBool(input)
+	if err != nil {
+		die(input + " is not a valid value for a boolean")
+	}
+	return val
+}
+
+func parseInt(input string) int {
+	val, err := strconv.ParseInt(input, 0, 64)
+	if err != nil {
+		die(input + " is not a valid value for an integer")
+	}
+	return int(val)
+}
+
+func parseUint(input string) int {
+	val, err := strconv.ParseUint(input, 0, 64)
+	if err != nil {
+		die(input + " is not a valid value for an unsigned integer")
+	}
+	return int(val)
+}
+
+func parsePort(input string) int {
+	port := parseUint(input)
+	if port < 1 || port > 65535 {
+		die(input + " is not a valid port\nExpected value between 1 and 65535")
+	}
+	return port
+}
+
+func validAddress(input string) {
+	tokens := strings.Split(input, ":")
+	if len(tokens) != 2 {
+		die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
+	}
+	matched, err := regexp.MatchString("^[a-zA-Z0-9]+([-a-zA-Z0-9.]+[-a-zA-Z0-9]+)?$", tokens[0])
+	die(err)
+	if !matched {
+		die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
+	}
+	parsePort(tokens[1])
+}
+
+func parseDeviceID(input string) protocol.DeviceID {
+	device, err := protocol.DeviceIDFromString(input)
+	if err != nil {
+		die(input + " is not a valid device id")
+	}
+	return device
+}

cmd/stcompdirs/main.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"crypto/md5"
 	"errors"
 	"flag"
 	"fmt"
@@ -14,8 +15,6 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-
-	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 func main() {
@@ -61,7 +60,7 @@ func compareDirectories(dirs ...string) error {
 				} else if res[i].name > res[0].name {
 					return fmt.Errorf("%s missing %v (present in %s)", dirs[i], res[0], dirs[0])
 				}
-				return fmt.Errorf("mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
+				return fmt.Errorf("Mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
 			}
 		}
 
@@ -75,7 +74,7 @@ type fileInfo struct {
 	name string
 	mode os.FileMode
 	mod  int64
-	hash [sha256.Size]byte
+	hash [16]byte
 }
 
 func (f fileInfo) String() string {
@@ -89,10 +88,10 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 		}
 
 		rn, _ := filepath.Rel(dir, path)
-		if rn == "." {
+		if rn == "." || rn == ".stfolder" {
 			return nil
 		}
-		if rn == ".stversions" || rn == ".stfolder" {
+		if rn == ".stversions" {
 			return filepath.SkipDir
 		}
 
@@ -107,7 +106,11 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 			if err != nil {
 				return err
 			}
-			f.hash = sha256.Sum256([]byte(tgt))
+			h := md5.New()
+			h.Write([]byte(tgt))
+			hash := h.Sum(nil)
+
+			copy(f.hash[:], hash)
 		} else if info.IsDir() {
 			f = fileInfo{
 				name: rn,
@@ -120,7 +123,7 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 				mode: info.Mode(),
 				mod:  info.ModTime().Unix(),
 			}
-			sum, err := sha256file(path)
+			sum, err := md5file(path)
 			if err != nil {
 				return err
 			}
@@ -147,14 +150,14 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
 	return errc
 }
 
-func sha256file(fname string) (hash [sha256.Size]byte, err error) {
+func md5file(fname string) (hash [16]byte, err error) {
 	f, err := os.Open(fname)
 	if err != nil {
 		return
 	}
 	defer f.Close()
 
-	h := sha256.New()
+	h := md5.New()
 	io.Copy(h, f)
 	hb := h.Sum(nil)
 	copy(hash[:], hb)

cmd/stcrashreceiver/diskstore.go

@@ -1,187 +0,0 @@
-// Copyright (C) 2023 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"io"
-	"log"
-	"os"
-	"path/filepath"
-	"sort"
-	"time"
-)
-
-type diskStore struct {
-	dir      string
-	inbox    chan diskEntry
-	maxBytes int64
-	maxFiles int
-
-	currentFiles []currentFile
-	currentSize  int64
-}
-
-type diskEntry struct {
-	path string
-	data []byte
-}
-
-type currentFile struct {
-	path  string
-	size  int64
-	mtime int64
-}
-
-func (d *diskStore) Serve(ctx context.Context) {
-	if err := os.MkdirAll(d.dir, 0750); err != nil {
-		log.Println("Creating directory:", err)
-		return
-	}
-
-	if err := d.inventory(); err != nil {
-		log.Println("Failed to inventory disk store:", err)
-	}
-	d.clean()
-
-	cleanTimer := time.NewTicker(time.Minute)
-	inventoryTimer := time.NewTicker(24 * time.Hour)
-
-	buf := new(bytes.Buffer)
-	gw := gzip.NewWriter(buf)
-	for {
-		select {
-		case entry := <-d.inbox:
-			path := d.fullPath(entry.path)
-
-			if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
-				log.Println("Creating directory:", err)
-				continue
-			}
-
-			buf.Reset()
-			gw.Reset(buf)
-			if _, err := gw.Write(entry.data); err != nil {
-				log.Println("Failed to compress crash report:", err)
-				continue
-			}
-			if err := gw.Close(); err != nil {
-				log.Println("Failed to compress crash report:", err)
-				continue
-			}
-			if err := os.WriteFile(path, buf.Bytes(), 0644); err != nil {
-				log.Printf("Failed to write %s: %v", entry.path, err)
-				_ = os.Remove(path)
-				continue
-			}
-
-			d.currentSize += int64(buf.Len())
-			d.currentFiles = append(d.currentFiles, currentFile{
-				size: int64(len(entry.data)),
-				path: path,
-			})
-
-		case <-cleanTimer.C:
-			d.clean()
-
-		case <-inventoryTimer.C:
-			if err := d.inventory(); err != nil {
-				log.Println("Failed to inventory disk store:", err)
-			}
-
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-func (d *diskStore) Put(path string, data []byte) bool {
-	select {
-	case d.inbox <- diskEntry{
-		path: path,
-		data: data,
-	}:
-		return true
-	default:
-		return false
-	}
-}
-
-func (d *diskStore) Get(path string) ([]byte, error) {
-	path = d.fullPath(path)
-	bs, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	gr, err := gzip.NewReader(bytes.NewReader(bs))
-	if err != nil {
-		return nil, err
-	}
-	defer gr.Close()
-	return io.ReadAll(gr)
-}
-
-func (d *diskStore) Exists(path string) bool {
-	path = d.fullPath(path)
-	_, err := os.Lstat(path)
-	return err == nil
-}
-
-func (d *diskStore) clean() {
-	for len(d.currentFiles) > 0 && (len(d.currentFiles) > d.maxFiles || d.currentSize > d.maxBytes) {
-		f := d.currentFiles[0]
-		log.Println("Removing", f.path)
-		if err := os.Remove(f.path); err != nil {
-			log.Println("Failed to remove file:", err)
-		}
-		d.currentFiles = d.currentFiles[1:]
-		d.currentSize -= f.size
-	}
-	var oldest time.Duration
-	if len(d.currentFiles) > 0 {
-		oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
-	}
-	log.Printf("Clean complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
-}
-
-func (d *diskStore) inventory() error {
-	d.currentFiles = nil
-	d.currentSize = 0
-	err := filepath.Walk(d.dir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if info.IsDir() {
-			return nil
-		}
-		if filepath.Ext(path) != ".gz" {
-			return nil
-		}
-		d.currentSize += info.Size()
-		d.currentFiles = append(d.currentFiles, currentFile{
-			path:  path,
-			size:  info.Size(),
-			mtime: info.ModTime().Unix(),
-		})
-		return nil
-	})
-	sort.Slice(d.currentFiles, func(i, j int) bool {
-		return d.currentFiles[i].mtime < d.currentFiles[j].mtime
-	})
-	var oldest time.Duration
-	if len(d.currentFiles) > 0 {
-		oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
-	}
-	log.Printf("Inventory complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
-	return err
-}
-
-func (d *diskStore) fullPath(path string) string {
-	return filepath.Join(d.dir, path[0:2], path[2:]) + ".gz"
-}

cmd/stcrashreceiver/main.go

@@ -1,154 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// Command stcrashreceiver is a trivial HTTP server that allows two things:
-//
-// - uploading files (crash reports) named like a SHA256 hash using a PUT request
-// - checking whether such file exists using a HEAD request
-//
-// Typically this should be deployed behind something that manages HTTPS.
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/alecthomas/kong"
-	"github.com/syncthing/syncthing/lib/sha256"
-	"github.com/syncthing/syncthing/lib/ur"
-
-	raven "github.com/getsentry/raven-go"
-)
-
-const maxRequestSize = 1 << 20 // 1 MiB
-
-type cli struct {
-	Dir           string        `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."`
-	DSN           string        `help:"Sentry DSN" env:"SENTRY_DSN"`
-	Listen        string        `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"`
-	MaxDiskFiles  int           `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"`
-	MaxDiskSizeMB int64         `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"`
-	CleanInterval time.Duration `help:"Interval between cleaning up old reports" default:"12h" env:"CLEAN_INTERVAL"`
-	SentryQueue   int           `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"`
-	DiskQueue     int           `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"`
-}
-
-func main() {
-	var params cli
-	kong.Parse(&params)
-
-	mux := http.NewServeMux()
-
-	ds := &diskStore{
-		dir:      filepath.Join(params.Dir, "crash_reports"),
-		inbox:    make(chan diskEntry, params.DiskQueue),
-		maxFiles: params.MaxDiskFiles,
-		maxBytes: params.MaxDiskSizeMB << 20,
-	}
-	go ds.Serve(context.Background())
-
-	ss := &sentryService{
-		dsn:   params.DSN,
-		inbox: make(chan sentryRequest, params.SentryQueue),
-	}
-	go ss.Serve(context.Background())
-
-	cr := &crashReceiver{
-		store:  ds,
-		sentry: ss,
-	}
-
-	mux.Handle("/", cr)
-	mux.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) {
-		w.Write([]byte("OK"))
-	})
-
-	if params.DSN != "" {
-		mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports")))
-	}
-
-	log.SetOutput(os.Stdout)
-	if err := http.ListenAndServe(params.Listen, mux); err != nil {
-		log.Fatalln("HTTP serve:", err)
-	}
-}
-
-func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *http.Request) {
-	return func(w http.ResponseWriter, req *http.Request) {
-		lr := io.LimitReader(req.Body, maxRequestSize)
-		bs, err := io.ReadAll(lr)
-		req.Body.Close()
-		if err != nil {
-			http.Error(w, err.Error(), 500)
-			return
-		}
-
-		var reports []ur.FailureReport
-		err = json.Unmarshal(bs, &reports)
-		if err != nil {
-			http.Error(w, err.Error(), 400)
-			return
-		}
-		if len(reports) == 0 {
-			// Shouldn't happen
-			log.Printf("Got zero failure reports")
-			return
-		}
-
-		version, err := parseVersion(reports[0].Version)
-		if err != nil {
-			http.Error(w, err.Error(), 400)
-			return
-		}
-		for _, r := range reports {
-			pkt := packet(version, "failure")
-			pkt.Message = r.Description
-			pkt.Extra = raven.Extra{
-				"count": r.Count,
-			}
-			for k, v := range r.Extra {
-				pkt.Extra[k] = v
-			}
-			if r.Goroutines != "" {
-				url, err := saveFailureWithGoroutines(r.FailureData, failureDir)
-				if err != nil {
-					log.Println("Saving failure report:", err)
-					http.Error(w, "Internal server error", http.StatusInternalServerError)
-					return
-				}
-				pkt.Extra["goroutinesURL"] = url
-			}
-			message := sanitizeMessageLDB(r.Description)
-			pkt.Fingerprint = []string{message}
-
-			if err := sendReport(dsn, pkt, userIDFor(req)); err != nil {
-				log.Println("Failed to send failure report:", err)
-			} else {
-				log.Println("Sent failure report:", r.Description)
-			}
-		}
-	}
-}
-
-func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string, error) {
-	bs := make([]byte, len(data.Description)+len(data.Goroutines))
-	copy(bs, data.Description)
-	copy(bs[len(data.Description):], data.Goroutines)
-	id := fmt.Sprintf("%x", sha256.Sum256(bs))
-	path := fullPathCompressed(failureDir, id)
-	err := compressAndWrite(bs, path)
-	if err != nil {
-		return "", err
-	}
-	return reportServer + path, nil
-}

cmd/stcrashreceiver/sentry.go

@@ -1,320 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"io"
-	"log"
-	"regexp"
-	"strings"
-	"sync"
-
-	raven "github.com/getsentry/raven-go"
-	"github.com/maruel/panicparse/v2/stack"
-)
-
-const reportServer = "https://crash.syncthing.net/report/"
-
-var loader = newGithubSourceCodeLoader()
-
-func init() {
-	raven.SetSourceCodeLoader(loader)
-}
-
-var (
-	clients    = make(map[string]*raven.Client)
-	clientsMut sync.Mutex
-)
-
-type sentryService struct {
-	dsn   string
-	inbox chan sentryRequest
-}
-
-type sentryRequest struct {
-	reportID string
-	data     []byte
-}
-
-func (s *sentryService) Serve(ctx context.Context) {
-	for {
-		select {
-		case req := <-s.inbox:
-			pkt, err := parseCrashReport(req.reportID, req.data)
-			if err != nil {
-				log.Println("Failed to parse crash report:", err)
-				continue
-			}
-			if err := sendReport(s.dsn, pkt, req.reportID); err != nil {
-				log.Println("Failed to send crash report:", err)
-			}
-
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-func (s *sentryService) Send(reportID string, data []byte) bool {
-	select {
-	case s.inbox <- sentryRequest{reportID, data}:
-		return true
-	default:
-		return false
-	}
-}
-
-func sendReport(dsn string, pkt *raven.Packet, userID string) error {
-	pkt.Interfaces = append(pkt.Interfaces, &raven.User{ID: userID})
-
-	clientsMut.Lock()
-	defer clientsMut.Unlock()
-
-	cli, ok := clients[dsn]
-	if !ok {
-		var err error
-		cli, err = raven.New(dsn)
-		if err != nil {
-			return err
-		}
-		clients[dsn] = cli
-	}
-
-	// The client sets release and such on the packet before sending, in the
-	// misguided idea that it knows this better than than the packet we give
-	// it. So we copy the values from the packet to the client first...
-	cli.SetRelease(pkt.Release)
-	cli.SetEnvironment(pkt.Environment)
-
-	defer cli.Wait()
-	_, errC := cli.Capture(pkt, nil)
-	return <-errC
-}
-
-func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
-	parts := bytes.SplitN(report, []byte("\n"), 2)
-	if len(parts) != 2 {
-		return nil, errors.New("no first line")
-	}
-
-	version, err := parseVersion(string(parts[0]))
-	if err != nil {
-		return nil, err
-	}
-	report = parts[1]
-
-	foundPanic := false
-	var subjectLine []byte
-	for {
-		parts = bytes.SplitN(report, []byte("\n"), 2)
-		if len(parts) != 2 {
-			return nil, errors.New("no panic line found")
-		}
-
-		line := parts[0]
-		report = parts[1]
-
-		if foundPanic {
-			// The previous line was our "Panic at ..." header. We are now
-			// at the beginning of the real panic trace and this is our
-			// subject line.
-			subjectLine = line
-			break
-		} else if bytes.HasPrefix(line, []byte("Panic at")) {
-			foundPanic = true
-		}
-	}
-
-	r := bytes.NewReader(report)
-	ctx, _, err := stack.ScanSnapshot(r, io.Discard, stack.DefaultOpts())
-	if err != nil && err != io.EOF {
-		return nil, err
-	}
-	if ctx == nil || len(ctx.Goroutines) == 0 {
-		return nil, errors.New("no goroutines found")
-	}
-
-	// Lock the source code loader to the version we are processing here.
-	if version.commit != "" {
-		// We have a commit hash, so we know exactly which source to use
-		loader.LockWithVersion(version.commit)
-	} else if strings.HasPrefix(version.tag, "v") {
-		// Lets hope the tag is close enough
-		loader.LockWithVersion(version.tag)
-	} else {
-		// Last resort
-		loader.LockWithVersion("main")
-	}
-	defer loader.Unlock()
-
-	var trace raven.Stacktrace
-	for _, gr := range ctx.Goroutines {
-		if gr.First {
-			trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
-			for i, sc := range gr.Stack.Calls {
-				trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name, sc.RemoteSrcPath, sc.Line, 3, nil)
-			}
-			break
-		}
-	}
-
-	pkt := packet(version, "crash")
-	pkt.Message = string(subjectLine)
-	pkt.Extra = raven.Extra{
-		"url": reportServer + path,
-	}
-	pkt.Interfaces = []raven.Interface{&trace}
-	pkt.Fingerprint = crashReportFingerprint(pkt.Message)
-
-	return pkt, nil
-}
-
-var (
-	indexRe          = regexp.MustCompile(`\[[-:0-9]+\]`)
-	sizeRe           = regexp.MustCompile(`(length|capacity) [0-9]+`)
-	ldbPosRe         = regexp.MustCompile(`(\(pos=)([0-9]+)\)`)
-	ldbChecksumRe    = regexp.MustCompile(`(want=0x)([a-z0-9]+)( got=0x)([a-z0-9]+)`)
-	ldbFileRe        = regexp.MustCompile(`(\[file=)([0-9]+)(\.ldb\])`)
-	ldbInternalKeyRe = regexp.MustCompile(`(internal key ")[^"]+(", len=)[0-9]+`)
-	ldbPathRe        = regexp.MustCompile(`(open|write|read) .+[\\/].+[\\/]index[^\\/]+[\\/][^\\/]+: `)
-)
-
-func sanitizeMessageLDB(message string) string {
-	message = ldbPosRe.ReplaceAllString(message, "${1}x)")
-	message = ldbFileRe.ReplaceAllString(message, "${1}x${3}")
-	message = ldbChecksumRe.ReplaceAllString(message, "${1}X${3}X")
-	message = ldbInternalKeyRe.ReplaceAllString(message, "${1}x${2}x")
-	message = ldbPathRe.ReplaceAllString(message, "$1 x: ")
-	return message
-}
-
-func crashReportFingerprint(message string) []string {
-	// Do not fingerprint on the stack in case of db corruption or fatal
-	// db io error - where it occurs doesn't matter.
-	orig := message
-	message = sanitizeMessageLDB(message)
-	if message != orig {
-		return []string{message}
-	}
-
-	message = indexRe.ReplaceAllString(message, "[x]")
-	message = sizeRe.ReplaceAllString(message, "$1 x")
-
-	// {{ default }} is what sentry uses as a fingerprint by default. While
-	// never specified, the docs point at this being some hash derived from the
-	// stack trace. Here we include the filtered panic message on top of that.
-	// https://docs.sentry.io/platforms/go/data-management/event-grouping/sdk-fingerprinting/#basic-example
-	return []string{"{{ default }}", message}
-}
-
-// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]
-// or, somewhere along the way the "+" in the version tag disappeared:
-// syncthing v1.23.7-dev.26.gdf7b56ae.dirty-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]
-var (
-	longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
-	gitExtraRE    = regexp.MustCompile(`\.\d+\.g[0-9a-f]+`) // ".1.g6aaae618"
-	gitExtraSepRE = regexp.MustCompile(`[.-]`)              // dot or dash
-)
-
-type version struct {
-	version  string   // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
-	tag      string   // "v1.1.4-rc.1"
-	commit   string   // "6aaae618", blank when absent
-	codename string   // "Erbium Earthworm"
-	runtime  string   // "go1.12.5"
-	goos     string   // "darwin"
-	goarch   string   // "amd64"
-	builder  string   // "jb@kvin.kastelo.net"
-	extra    []string // "foo", "bar"
-}
-
-func (v version) environment() string {
-	if v.commit != "" {
-		return "Development"
-	}
-	if strings.Contains(v.tag, "-rc.") {
-		return "Candidate"
-	}
-	if strings.Contains(v.tag, "-") {
-		return "Beta"
-	}
-	return "Stable"
-}
-
-func parseVersion(line string) (version, error) {
-	m := longVersionRE.FindStringSubmatch(line)
-	if len(m) == 0 {
-		return version{}, errors.New("unintelligeble version string")
-	}
-
-	v := version{
-		version:  m[1],
-		codename: m[2],
-		runtime:  m[3],
-		goos:     m[4],
-		goarch:   m[5],
-		builder:  m[6],
-	}
-
-	// Split the version tag into tag and commit. This is old style
-	// v1.2.3-something.4+11-g12345678 or newer with just dots
-	// v1.2.3-something.4.11.g12345678 or v1.2.3-dev.11.g12345678.
-	parts := []string{v.version}
-	if strings.Contains(v.version, "+") {
-		parts = strings.Split(v.version, "+")
-	} else {
-		idxs := gitExtraRE.FindStringIndex(v.version)
-		if len(idxs) > 0 {
-			parts = []string{v.version[:idxs[0]], v.version[idxs[0]+1:]}
-		}
-	}
-	v.tag = parts[0]
-	if len(parts) > 1 {
-		fields := gitExtraSepRE.Split(parts[1], -1)
-		if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
-			v.commit = fields[1][1:]
-		}
-	}
-
-	if len(m) >= 8 && m[7] != "" {
-		tags := strings.Split(m[7], ",")
-		for i := range tags {
-			tags[i] = strings.TrimSpace(tags[i])
-		}
-		v.extra = tags
-	}
-
-	return v, nil
-}
-
-func packet(version version, reportType string) *raven.Packet {
-	pkt := &raven.Packet{
-		Platform:    "go",
-		Release:     version.tag,
-		Environment: version.environment(),
-		Tags: raven.Tags{
-			raven.Tag{Key: "version", Value: version.version},
-			raven.Tag{Key: "tag", Value: version.tag},
-			raven.Tag{Key: "codename", Value: version.codename},
-			raven.Tag{Key: "runtime", Value: version.runtime},
-			raven.Tag{Key: "goos", Value: version.goos},
-			raven.Tag{Key: "goarch", Value: version.goarch},
-			raven.Tag{Key: "builder", Value: version.builder},
-			raven.Tag{Key: "report_type", Value: reportType},
-		},
-	}
-	if version.commit != "" {
-		pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
-	}
-	for _, tag := range version.extra {
-		pkt.Tags = append(pkt.Tags, raven.Tag{Key: tag, Value: "1"})
-	}
-	return pkt
-}

cmd/stcrashreceiver/sentry_test.go

@@ -1,155 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"fmt"
-	"os"
-	"testing"
-)
-
-func TestParseVersion(t *testing.T) {
-	cases := []struct {
-		longVersion string
-		parsed      version
-	}{
-		{
-			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`,
-			parsed: version{
-				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
-				tag:      "v1.1.4-rc.1",
-				commit:   "6aaae618",
-				codename: "Erbium Earthworm",
-				runtime:  "go1.12.5",
-				goos:     "darwin",
-				goarch:   "amd64",
-				builder:  "jb@kvin.kastelo.net",
-			},
-		},
-		{
-			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]`,
-			parsed: version{
-				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
-				tag:      "v1.1.4-rc.1",
-				commit:   "6aaae618",
-				codename: "Erbium Earthworm",
-				runtime:  "go1.12.5",
-				goos:     "darwin",
-				goarch:   "amd64",
-				builder:  "jb@kvin.kastelo.net",
-				extra:    []string{"foo", "bar"},
-			},
-		},
-		{
-			longVersion: `syncthing v1.23.7-dev.26.gdf7b56ae-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]`,
-			parsed: version{
-				version:  "v1.23.7-dev.26.gdf7b56ae-stversionextra",
-				tag:      "v1.23.7-dev",
-				commit:   "df7b56ae",
-				codename: "Fermium Flea",
-				runtime:  "go1.20.5",
-				goos:     "darwin",
-				goarch:   "arm64",
-				builder:  "jb@ok.kastelo.net",
-				extra:    []string{"Some Wrapper", "purego", "stnoupgrade"},
-			},
-		},
-	}
-
-	for _, tc := range cases {
-		v, err := parseVersion(tc.longVersion)
-		if err != nil {
-			t.Errorf("%s\nerror: %v\n", tc.longVersion, err)
-			continue
-		}
-		if fmt.Sprint(v) != fmt.Sprint(tc.parsed) {
-			t.Errorf("%s\nA: %v\nE: %v\n", tc.longVersion, v, tc.parsed)
-		}
-	}
-}
-
-func TestParseReport(t *testing.T) {
-	bs, err := os.ReadFile("_testdata/panic.log")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	pkt, err := parseCrashReport("1/2/345", bs)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	bs, err = pkt.JSON()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	fmt.Printf("%s\n", bs)
-}
-
-func TestCrashReportFingerprint(t *testing.T) {
-	cases := []struct {
-		message, exp string
-		ldb          bool
-	}{
-		{
-			message: "panic: leveldb/table: corruption on data-block (pos=51308946): checksum mismatch, want=0xa89f9aa0 got=0xd27cc4c7 [file=004003.ldb]",
-			exp:     "panic: leveldb/table: corruption on data-block (pos=x): checksum mismatch, want=0xX got=0xX [file=x.ldb]",
-			ldb:     true,
-		},
-		{
-			message: "panic: leveldb/table: corruption on table-footer (pos=248): bad magic number [file=001370.ldb]",
-			exp:     "panic: leveldb/table: corruption on table-footer (pos=x): bad magic number [file=x.ldb]",
-			ldb:     true,
-		},
-		{
-			message: "panic: runtime error: slice bounds out of range [4294967283:4194304]",
-			exp:     "panic: runtime error: slice bounds out of range [x]",
-		},
-		{
-			message: "panic: runtime error: slice bounds out of range [-2:]",
-			exp:     "panic: runtime error: slice bounds out of range [x]",
-		},
-		{
-			message: "panic: runtime error: slice bounds out of range [:4294967283] with capacity 32768",
-			exp:     "panic: runtime error: slice bounds out of range [x] with capacity x",
-		},
-		{
-			message: "panic: runtime error: index out of range [0] with length 0",
-			exp:     "panic: runtime error: index out of range [x] with length x",
-		},
-		{
-			message: `panic: leveldb: internal key "\x01", len=1: invalid length`,
-			exp:     `panic: leveldb: internal key "x", len=x: invalid length`,
-			ldb:     true,
-		},
-		{
-			message: `panic: write /var/syncthing/config/index-v0.14.0.db/2732813.log: cannot allocate memory`,
-			exp:     `panic: write x: cannot allocate memory`,
-			ldb:     true,
-		},
-		{
-			message: `panic: filling Blocks: read C:\Users\Serv-Resp-Tizayuca\AppData\Local\Syncthing\index-v0.14.0.db\006561.ldb: Error de datos (comprobación de redundancia cíclica).`,
-			exp:     `panic: filling Blocks: read x: Error de datos (comprobación de redundancia cíclica).`,
-			ldb:     true,
-		},
-	}
-
-	for i, tc := range cases {
-		fingerprint := crashReportFingerprint(tc.message)
-
-		expLen := 2
-		if tc.ldb {
-			expLen = 1
-		}
-		if l := len(fingerprint); l != expLen {
-			t.Errorf("tc %v: Unexpected fingerprint length: %v != %v", i, l, expLen)
-		} else if msg := fingerprint[expLen-1]; msg != tc.exp {
-			t.Errorf("tc %v:\n\"%v\" !=\n\"%v\"", i, msg, tc.exp)
-		}
-	}
-}

cmd/stcrashreceiver/sourcecodeloader.go

@@ -1,118 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"net/http"
-	"path/filepath"
-	"strings"
-	"sync"
-	"time"
-)
-
-const (
-	urlPrefix   = "https://raw.githubusercontent.com/syncthing/syncthing/"
-	httpTimeout = 10 * time.Second
-)
-
-type githubSourceCodeLoader struct {
-	mut     sync.Mutex
-	version string
-	cache   map[string]map[string][][]byte // version -> file -> lines
-	client  *http.Client
-}
-
-func newGithubSourceCodeLoader() *githubSourceCodeLoader {
-	return &githubSourceCodeLoader{
-		cache:  make(map[string]map[string][][]byte),
-		client: &http.Client{Timeout: httpTimeout},
-	}
-}
-
-func (l *githubSourceCodeLoader) LockWithVersion(version string) {
-	l.mut.Lock()
-	l.version = version
-	if _, ok := l.cache[version]; !ok {
-		l.cache[version] = make(map[string][][]byte)
-	}
-}
-
-func (l *githubSourceCodeLoader) Unlock() {
-	l.mut.Unlock()
-}
-
-func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]byte, int) {
-	filename = filepath.ToSlash(filename)
-	lines, ok := l.cache[l.version][filename]
-	if !ok {
-		// Cache whatever we managed to find (or nil if nothing, so we don't try again)
-		defer func() {
-			l.cache[l.version][filename] = lines
-		}()
-
-		knownPrefixes := []string{"/lib/", "/cmd/"}
-		var idx int
-		for _, pref := range knownPrefixes {
-			idx = strings.Index(filename, pref)
-			if idx >= 0 {
-				break
-			}
-		}
-		if idx == -1 {
-			return nil, 0
-		}
-
-		url := urlPrefix + l.version + filename[idx:]
-		resp, err := l.client.Get(url)
-
-		if err != nil {
-			fmt.Println("Loading source:", err)
-			return nil, 0
-		}
-		if resp.StatusCode != http.StatusOK {
-			fmt.Println("Loading source:", resp.Status)
-			return nil, 0
-		}
-		data, err := io.ReadAll(resp.Body)
-		_ = resp.Body.Close()
-		if err != nil {
-			fmt.Println("Loading source:", err.Error())
-			return nil, 0
-		}
-		lines = bytes.Split(data, []byte{'\n'})
-	}
-
-	return getLineFromLines(lines, line, context)
-}
-
-func getLineFromLines(lines [][]byte, line, context int) ([][]byte, int) {
-	if lines == nil {
-		// cached error from ReadFile: return no lines
-		return nil, 0
-	}
-
-	line-- // stack trace lines are 1-indexed
-	start := line - context
-	var idx int
-	if start < 0 {
-		start = 0
-		idx = line
-	} else {
-		idx = context
-	}
-	end := line + context + 1
-	if line >= len(lines) {
-		return nil, 0
-	}
-	if end > len(lines) {
-		end = len(lines)
-	}
-	return lines[start:end], idx
-}

cmd/stcrashreceiver/stcrashreceiver.go

@@ -1,93 +0,0 @@
-// Copyright (C) 2019 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"io"
-	"log"
-	"net/http"
-	"path"
-	"strings"
-)
-
-type crashReceiver struct {
-	store  *diskStore
-	sentry *sentryService
-}
-
-func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	// The final path component should be a SHA256 hash in hex, so 64 hex
-	// characters. We don't care about case on the request but use lower
-	// case internally.
-	reportID := strings.ToLower(path.Base(req.URL.Path))
-	if len(reportID) != 64 {
-		http.Error(w, "Bad request", http.StatusBadRequest)
-		return
-	}
-	for _, c := range reportID {
-		if c >= 'a' && c <= 'f' {
-			continue
-		}
-		if c >= '0' && c <= '9' {
-			continue
-		}
-		http.Error(w, "Bad request", http.StatusBadRequest)
-		return
-	}
-
-	switch req.Method {
-	case http.MethodGet:
-		r.serveGet(reportID, w, req)
-	case http.MethodHead:
-		r.serveHead(reportID, w, req)
-	case http.MethodPut:
-		r.servePut(reportID, w, req)
-	default:
-		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-
-// serveGet responds to GET requests by serving the uncompressed report.
-func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http.Request) {
-	bs, err := r.store.Get(reportID)
-	if err != nil {
-		http.Error(w, "Not found", http.StatusNotFound)
-		return
-	}
-	w.Write(bs)
-}
-
-// serveHead responds to HEAD requests by checking if the named report
-// already exists in the system.
-func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) {
-	if !r.store.Exists(reportID) {
-		http.Error(w, "Not found", http.StatusNotFound)
-	}
-}
-
-// servePut accepts and stores the given report.
-func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *http.Request) {
-	// Read at most maxRequestSize of report data.
-	log.Println("Receiving report", reportID)
-	lr := io.LimitReader(req.Body, maxRequestSize)
-	bs, err := io.ReadAll(lr)
-	if err != nil {
-		log.Println("Reading report:", err)
-		http.Error(w, "Internal server error", http.StatusInternalServerError)
-		return
-	}
-
-	// Store the report
-	if !r.store.Put(reportID, bs) {
-		log.Println("Failed to store report (queue full):", reportID)
-	}
-
-	// Send the report to Sentry
-	if !r.sentry.Send(reportID, bs) {
-		log.Println("Failed to send report to sentry (queue full):", reportID)
-	}
-}

cmd/stcrashreceiver/util.go

@@ -1,57 +0,0 @@
-// Copyright (C) 2021 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"net"
-	"net/http"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/sha256"
-)
-
-// userIDFor returns a string we can use as the user ID for the purpose of
-// counting affected users. It's the truncated hash of a salt, the user
-// remote IP, and the current month.
-func userIDFor(req *http.Request) string {
-	addr := req.RemoteAddr
-	if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
-		addr = fwd
-	}
-	if host, _, err := net.SplitHostPort(addr); err == nil {
-		addr = host
-	}
-	now := time.Now().Format("200601")
-	salt := "stcrashreporter"
-	hash := sha256.Sum256([]byte(salt + addr + now))
-	return fmt.Sprintf("%x", hash[:8])
-}
-
-// 01234567890abcdef... => 01/23
-func dirFor(base string) string {
-	return filepath.Join(base[0:2], base[2:4])
-}
-
-func fullPathCompressed(root, reportID string) string {
-	return filepath.Join(root, dirFor(reportID), reportID) + ".gz"
-}
-
-func compressAndWrite(bs []byte, fullPath string) error {
-	// Compress the report for storage
-	buf := new(bytes.Buffer)
-	gw := gzip.NewWriter(buf)
-	_, _ = gw.Write(bs) // can't fail
-	gw.Close()
-
-	// Create an output file with the compressed report
-	return os.WriteFile(fullPath, buf.Bytes(), 0644)
-}

cmd/stdisco/main.go

@@ -7,7 +7,6 @@
 package main
 
 import (
-	"context"
 	"crypto/rand"
 	"encoding/binary"
 	"flag"
@@ -48,16 +47,14 @@ func main() {
 		log.Println("My ID:", myID)
 	}
 
-	ctx := context.Background()
-
-	runbeacon(ctx, beacon.NewMulticast(mc), fake)
-	runbeacon(ctx, beacon.NewBroadcast(bc), fake)
+	runbeacon(beacon.NewMulticast(mc), fake)
+	runbeacon(beacon.NewBroadcast(bc), fake)
 
 	select {}
 }
 
-func runbeacon(ctx context.Context, bc beacon.Interface, fake bool) {
-	go bc.Serve(ctx)
+func runbeacon(bc beacon.Interface, fake bool) {
+	go bc.Serve()
 	go recv(bc)
 	if fake {
 		go send(bc)

cmd/stdiscosrv/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2014-2015 The Discosrv Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+- The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cmd/stdiscosrv/README.md

@@ -6,5 +6,33 @@ This is the global discovery server for the `syncthing` project.
 Usage
 -----
 
-https://docs.syncthing.net/users/stdiscosrv.html
+The discovery server supports `ql` and `postgres` backends.
+Specify the backend via `-db-backend` and the database DSN via `-db-dsn`.
 
+By default it will use in-memory `ql` backend. If you wish to persist the
+information on disk between restarts in `ql`, specify a file DSN:
+
+```bash
+$ stdiscosrv -db-dsn="file:///var/run/stdiscosrv.db"
+```
+
+For `postgres`, you will need to create a database and a user with permissions
+to create tables in it, then start the stdiscosrv as follows:
+
+```bash
+$ export STDISCOSRV_DB_DSN="postgres://user:password@localhost/databasename"
+$ stdiscosrv -db-backend="postgres"
+```
+
+You can pass the DSN as command line option, but the value what you pass in will
+be visible in most process managers, potentially exposing the database password
+to other users.
+
+In all cases, the appropriate tables and indexes will be created at first
+startup. If it doesn't exit with an error, you're fine.
+
+See `stdiscosrv -help` for other options.
+
+##### Third-party attribution
+
+[cznic/lldb](https://github.com/cznic/lldb), Copyright (C) 2014 The lldb Authors.

cmd/stdiscosrv/apisrv.go

@@ -1,487 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"crypto/tls"
-	"encoding/base64"
-	"encoding/json"
-	"encoding/pem"
-	"errors"
-	"fmt"
-	io "io"
-	"log"
-	"math/rand"
-	"net"
-	"net/http"
-	"net/url"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/stringutil"
-)
-
-// announcement is the format received from and sent to clients
-type announcement struct {
-	Seen      time.Time `json:"seen"`
-	Addresses []string  `json:"addresses"`
-}
-
-type apiSrv struct {
-	addr     string
-	cert     tls.Certificate
-	db       database
-	listener net.Listener
-	repl     replicator // optional
-	useHTTP  bool
-
-	mapsMut sync.Mutex
-	misses  map[string]int32
-}
-
-type requestID int64
-
-func (i requestID) String() string {
-	return fmt.Sprintf("%016x", int64(i))
-}
-
-type contextKey int
-
-const idKey contextKey = iota
-
-func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP bool) *apiSrv {
-	return &apiSrv{
-		addr:    addr,
-		cert:    cert,
-		db:      db,
-		repl:    repl,
-		useHTTP: useHTTP,
-		misses:  make(map[string]int32),
-	}
-}
-
-func (s *apiSrv) Serve(_ context.Context) error {
-	if s.useHTTP {
-		listener, err := net.Listen("tcp", s.addr)
-		if err != nil {
-			log.Println("Listen:", err)
-			return err
-		}
-		s.listener = listener
-	} else {
-		tlsCfg := &tls.Config{
-			Certificates: []tls.Certificate{s.cert},
-			ClientAuth:   tls.RequestClientCert,
-			MinVersion:   tls.VersionTLS12,
-			NextProtos:   []string{"h2", "http/1.1"},
-		}
-
-		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
-		if err != nil {
-			log.Println("Listen:", err)
-			return err
-		}
-		s.listener = tlsListener
-	}
-
-	http.HandleFunc("/", s.handler)
-	http.HandleFunc("/ping", handlePing)
-
-	srv := &http.Server{
-		ReadTimeout:    httpReadTimeout,
-		WriteTimeout:   httpWriteTimeout,
-		MaxHeaderBytes: httpMaxHeaderBytes,
-		ErrorLog:       log.New(io.Discard, "", 0),
-	}
-
-	err := srv.Serve(s.listener)
-	if err != nil {
-		log.Println("Serve:", err)
-	}
-	return err
-}
-
-func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
-	t0 := time.Now()
-
-	lw := NewLoggingResponseWriter(w)
-
-	defer func() {
-		diff := time.Since(t0)
-		apiRequestsSeconds.WithLabelValues(req.Method).Observe(diff.Seconds())
-		apiRequestsTotal.WithLabelValues(req.Method, strconv.Itoa(lw.statusCode)).Inc()
-	}()
-
-	reqID := requestID(rand.Int63())
-	req = req.WithContext(context.WithValue(req.Context(), idKey, reqID))
-
-	if debug {
-		log.Println(reqID, req.Method, req.URL, req.Proto)
-	}
-
-	remoteAddr := &net.TCPAddr{
-		IP:   nil,
-		Port: -1,
-	}
-
-	if s.useHTTP {
-		remoteAddr.IP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
-		if parsedPort, err := strconv.ParseInt(req.Header.Get("X-Client-Port"), 10, 0); err == nil {
-			remoteAddr.Port = int(parsedPort)
-		}
-	} else {
-		var err error
-		remoteAddr, err = net.ResolveTCPAddr("tcp", req.RemoteAddr)
-		if err != nil {
-			log.Println("remoteAddr:", err)
-			lw.Header().Set("Retry-After", errorRetryAfterString())
-			http.Error(lw, "Internal Server Error", http.StatusInternalServerError)
-			apiRequestsTotal.WithLabelValues("no_remote_addr").Inc()
-			return
-		}
-	}
-
-	switch req.Method {
-	case http.MethodGet:
-		s.handleGET(lw, req)
-	case http.MethodPost:
-		s.handlePOST(remoteAddr, lw, req)
-	default:
-		http.Error(lw, "Method Not Allowed", http.StatusMethodNotAllowed)
-	}
-}
-
-func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
-	reqID := req.Context().Value(idKey).(requestID)
-
-	deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
-	if err != nil {
-		if debug {
-			log.Println(reqID, "bad device param")
-		}
-		lookupRequestsTotal.WithLabelValues("bad_request").Inc()
-		w.Header().Set("Retry-After", errorRetryAfterString())
-		http.Error(w, "Bad Request", http.StatusBadRequest)
-		return
-	}
-
-	key := deviceID.String()
-	rec, err := s.db.get(key)
-	if err != nil {
-		// some sort of internal error
-		lookupRequestsTotal.WithLabelValues("internal_error").Inc()
-		w.Header().Set("Retry-After", errorRetryAfterString())
-		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-		return
-	}
-
-	if len(rec.Addresses) == 0 {
-		lookupRequestsTotal.WithLabelValues("not_found").Inc()
-
-		s.mapsMut.Lock()
-		misses := s.misses[key]
-		if misses < rec.Misses {
-			misses = rec.Misses + 1
-		} else {
-			misses++
-		}
-		s.misses[key] = misses
-		s.mapsMut.Unlock()
-
-		if misses%notFoundMissesWriteInterval == 0 {
-			rec.Misses = misses
-			rec.Missed = time.Now().UnixNano()
-			rec.Addresses = nil
-			// rec.Seen retained from get
-			s.db.put(key, rec)
-		}
-
-		w.Header().Set("Retry-After", notFoundRetryAfterString(int(misses)))
-		http.Error(w, "Not Found", http.StatusNotFound)
-		return
-	}
-
-	lookupRequestsTotal.WithLabelValues("success").Inc()
-
-	w.Header().Set("Content-Type", "application/json")
-	var bw io.Writer = w
-
-	// Use compression if the client asks for it
-	if strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
-		w.Header().Set("Content-Encoding", "gzip")
-		gw := gzip.NewWriter(bw)
-		defer gw.Close()
-		bw = gw
-	}
-
-	json.NewEncoder(bw).Encode(announcement{
-		Seen:      time.Unix(0, rec.Seen).Truncate(time.Second),
-		Addresses: addressStrs(rec.Addresses),
-	})
-}
-
-func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req *http.Request) {
-	reqID := req.Context().Value(idKey).(requestID)
-
-	rawCert, err := certificateBytes(req)
-	if err != nil {
-		if debug {
-			log.Println(reqID, "no certificates:", err)
-		}
-		announceRequestsTotal.WithLabelValues("no_certificate").Inc()
-		w.Header().Set("Retry-After", errorRetryAfterString())
-		http.Error(w, "Forbidden", http.StatusForbidden)
-		return
-	}
-
-	var ann announcement
-	if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
-		if debug {
-			log.Println(reqID, "decode:", err)
-		}
-		announceRequestsTotal.WithLabelValues("bad_request").Inc()
-		w.Header().Set("Retry-After", errorRetryAfterString())
-		http.Error(w, "Bad Request", http.StatusBadRequest)
-		return
-	}
-
-	deviceID := protocol.NewDeviceID(rawCert)
-
-	addresses := fixupAddresses(remoteAddr, ann.Addresses)
-	if len(addresses) == 0 {
-		announceRequestsTotal.WithLabelValues("bad_request").Inc()
-		w.Header().Set("Retry-After", errorRetryAfterString())
-		http.Error(w, "Bad Request", http.StatusBadRequest)
-		return
-	}
-
-	if err := s.handleAnnounce(deviceID, addresses); err != nil {
-		announceRequestsTotal.WithLabelValues("internal_error").Inc()
-		w.Header().Set("Retry-After", errorRetryAfterString())
-		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-		return
-	}
-
-	announceRequestsTotal.WithLabelValues("success").Inc()
-
-	w.Header().Set("Reannounce-After", reannounceAfterString())
-	w.WriteHeader(http.StatusNoContent)
-}
-
-func (s *apiSrv) Stop() {
-	s.listener.Close()
-}
-
-func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string) error {
-	key := deviceID.String()
-	now := time.Now()
-	expire := now.Add(addressExpiryTime).UnixNano()
-
-	dbAddrs := make([]DatabaseAddress, len(addresses))
-	for i := range addresses {
-		dbAddrs[i].Address = addresses[i]
-		dbAddrs[i].Expires = expire
-	}
-
-	// The address slice must always be sorted for database merges to work
-	// properly.
-	sort.Sort(databaseAddressOrder(dbAddrs))
-
-	seen := now.UnixNano()
-	if s.repl != nil {
-		s.repl.send(key, dbAddrs, seen)
-	}
-	return s.db.merge(key, dbAddrs, seen)
-}
-
-func handlePing(w http.ResponseWriter, _ *http.Request) {
-	w.WriteHeader(204)
-}
-
-func certificateBytes(req *http.Request) ([]byte, error) {
-	if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
-		return req.TLS.PeerCertificates[0].Raw, nil
-	}
-
-	var bs []byte
-
-	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
-		if strings.Contains(hdr, "%") {
-			// Nginx using $ssl_client_escaped_cert
-			// The certificate is in PEM format with url encoding.
-			// We need to decode for the PEM decoder
-			hdr, err := url.QueryUnescape(hdr)
-			if err != nil {
-				// Decoding failed
-				return nil, err
-			}
-
-			bs = []byte(hdr)
-		} else {
-			// Nginx using $ssl_client_cert
-			// The certificate is in PEM format but with spaces for newlines. We
-			// need to reinstate the newlines for the PEM decoder. But we need to
-			// leave the spaces in the BEGIN and END lines - the first and last
-			// space - alone.
-			bs = []byte(hdr)
-			firstSpace := bytes.Index(bs, []byte(" "))
-			lastSpace := bytes.LastIndex(bs, []byte(" "))
-			for i := firstSpace + 1; i < lastSpace; i++ {
-				if bs[i] == ' ' {
-					bs[i] = '\n'
-				}
-			}
-		}
-	} else if hdr := req.Header.Get("X-Tls-Client-Cert-Der-Base64"); hdr != "" {
-		// Caddy {tls_client_certificate_der_base64}
-		hdr, err := base64.StdEncoding.DecodeString(hdr)
-		if err != nil {
-			// Decoding failed
-			return nil, err
-		}
-
-		bs = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: hdr})
-	} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
-		// Traefik 2 passtlsclientcert
-		// The certificate is in PEM format with url encoding but without newlines
-		// and start/end statements. We need to decode, reinstate the newlines every 64
-		// character and add statements for the PEM decoder
-		hdr, err := url.QueryUnescape(hdr)
-		if err != nil {
-			// Decoding failed
-			return nil, err
-		}
-
-		for i := 64; i < len(hdr); i += 65 {
-			hdr = hdr[:i] + "\n" + hdr[i:]
-		}
-
-		hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
-		hdr = hdr + "\n-----END CERTIFICATE-----\n"
-		bs = []byte(hdr)
-	}
-
-	if bs == nil {
-		return nil, errors.New("empty certificate header")
-	}
-
-	block, _ := pem.Decode(bs)
-	if block == nil {
-		// Decoding failed
-		return nil, errors.New("certificate decode result is empty")
-	}
-
-	return block.Bytes, nil
-}
-
-// fixupAddresses checks the list of addresses, removing invalid ones and
-// replacing unspecified IPs with the given remote IP.
-func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
-	fixed := make([]string, 0, len(addresses))
-	for _, annAddr := range addresses {
-		uri, err := url.Parse(annAddr)
-		if err != nil {
-			continue
-		}
-
-		host, port, err := net.SplitHostPort(uri.Host)
-		if err != nil {
-			continue
-		}
-
-		ip := net.ParseIP(host)
-
-		// Some classes of IP are no-go.
-		if ip.IsLoopback() || ip.IsMulticast() {
-			continue
-		}
-
-		if remote != nil {
-			if host == "" || ip.IsUnspecified() {
-				// Replace the unspecified IP with the request source.
-
-				// ... unless the request source is the loopback address or
-				// multicast/unspecified (can't happen, really).
-				if remote.IP.IsLoopback() || remote.IP.IsMulticast() || remote.IP.IsUnspecified() {
-					continue
-				}
-
-				// Do not use IPv6 remote address if requested scheme is ...4
-				// (i.e., tcp4, etc.)
-				if strings.HasSuffix(uri.Scheme, "4") && remote.IP.To4() == nil {
-					continue
-				}
-
-				// Do not use IPv4 remote address if requested scheme is ...6
-				if strings.HasSuffix(uri.Scheme, "6") && remote.IP.To4() != nil {
-					continue
-				}
-
-				host = remote.IP.String()
-			}
-
-			// If zero port was specified, use remote port.
-			if port == "0" && remote.Port > 0 {
-				port = strconv.Itoa(remote.Port)
-			}
-		}
-
-		uri.Host = net.JoinHostPort(host, port)
-		fixed = append(fixed, uri.String())
-	}
-
-	// Remove duplicate addresses
-	fixed = stringutil.UniqueTrimmedStrings(fixed)
-
-	return fixed
-}
-
-type loggingResponseWriter struct {
-	http.ResponseWriter
-	statusCode int
-}
-
-func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
-	return &loggingResponseWriter{w, http.StatusOK}
-}
-
-func (lrw *loggingResponseWriter) WriteHeader(code int) {
-	lrw.statusCode = code
-	lrw.ResponseWriter.WriteHeader(code)
-}
-
-func addressStrs(dbAddrs []DatabaseAddress) []string {
-	res := make([]string, len(dbAddrs))
-	for i, a := range dbAddrs {
-		res[i] = a.Address
-	}
-	return res
-}
-
-func errorRetryAfterString() string {
-	return strconv.Itoa(errorRetryAfterSeconds + rand.Intn(errorRetryFuzzSeconds))
-}
-
-func notFoundRetryAfterString(misses int) string {
-	retryAfterS := notFoundRetryMinSeconds + notFoundRetryIncSeconds*misses
-	if retryAfterS > notFoundRetryMaxSeconds {
-		retryAfterS = notFoundRetryMaxSeconds
-	}
-	retryAfterS += rand.Intn(notFoundRetryFuzzSeconds)
-	return strconv.Itoa(retryAfterS)
-}
-
-func reannounceAfterString() string {
-	return strconv.Itoa(reannounceAfterSeconds + rand.Intn(reannounzeFuzzSeconds))
-}

cmd/stdiscosrv/apisrv_test.go

@@ -1,88 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"fmt"
-	"net"
-	"testing"
-)
-
-func TestFixupAddresses(t *testing.T) {
-	cases := []struct {
-		remote *net.TCPAddr
-		in     []string
-		out    []string
-	}{
-		{ // verbatim passthrough
-			in:  []string{"tcp://1.2.3.4:22000"},
-			out: []string{"tcp://1.2.3.4:22000"},
-		}, { // unspecified replaced by remote
-			remote: addr("1.2.3.4", 22000),
-			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
-			out:    []string{"tcp://1.2.3.4:22000", "tcp://192.0.2.42:22000"},
-		}, { // unspecified not used as replacement
-			remote: addr("0.0.0.0", 22000),
-			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
-			out:    []string{"tcp://192.0.2.42:22000"},
-		}, { // unspecified not used as replacement
-			remote: addr("::", 22000),
-			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
-			out:    []string{"tcp://192.0.2.42:22000"},
-		}, { // localhost not used as replacement
-			remote: addr("127.0.0.1", 22000),
-			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
-			out:    []string{"tcp://192.0.2.42:22000"},
-		}, { // localhost not used as replacement
-			remote: addr("::1", 22000),
-			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
-			out:    []string{"tcp://192.0.2.42:22000"},
-		}, { // multicast not used as replacement
-			remote: addr("224.0.0.1", 22000),
-			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
-			out:    []string{"tcp://192.0.2.42:22000"},
-		}, { // multicast not used as replacement
-			remote: addr("ff80::42", 22000),
-			in:     []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
-			out:    []string{"tcp://192.0.2.42:22000"},
-		}, { // explicitly announced weirdness is also filtered
-			remote: addr("192.0.2.42", 22000),
-			in:     []string{"tcp://:22000", "tcp://127.1.2.3:22000", "tcp://[::1]:22000", "tcp://[ff80::42]:22000"},
-			out:    []string{"tcp://192.0.2.42:22000"},
-		}, { // port remapping
-			remote: addr("123.123.123.123", 9000),
-			in:     []string{"tcp://0.0.0.0:0"},
-			out:    []string{"tcp://123.123.123.123:9000"},
-		}, { // unspecified port remapping
-			remote: addr("123.123.123.123", 9000),
-			in:     []string{"tcp://:0"},
-			out:    []string{"tcp://123.123.123.123:9000"},
-		}, { // empty remapping
-			remote: addr("123.123.123.123", 9000),
-			in:     []string{"tcp://"},
-			out:    []string{},
-		}, { // port only remapping
-			remote: addr("123.123.123.123", 9000),
-			in:     []string{"tcp://44.44.44.44:0"},
-			out:    []string{"tcp://44.44.44.44:9000"},
-		},
-	}
-
-	for _, tc := range cases {
-		out := fixupAddresses(tc.remote, tc.in)
-		if fmt.Sprint(out) != fmt.Sprint(tc.out) {
-			t.Errorf("fixupAddresses(%v, %v) => %v, expected %v", tc.remote, tc.in, out, tc.out)
-		}
-	}
-}
-
-func addr(host string, port int) *net.TCPAddr {
-	return &net.TCPAddr{
-		IP:   net.ParseIP(host),
-		Port: port,
-	}
-}

cmd/stdiscosrv/clean.go

@@ -0,0 +1,75 @@
+// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"database/sql"
+	"log"
+	"time"
+)
+
+type cleansrv struct {
+	intv time.Duration
+	db   *sql.DB
+	prep map[string]*sql.Stmt
+}
+
+func (s *cleansrv) Serve() {
+	for {
+		time.Sleep(next(s.intv))
+
+		err := s.cleanOldEntries()
+		if err != nil {
+			log.Println("Clean:", err)
+		}
+	}
+}
+
+func (s *cleansrv) Stop() {
+	panic("stop unimplemented")
+}
+
+func (s *cleansrv) cleanOldEntries() (err error) {
+	var tx *sql.Tx
+	tx, err = s.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err == nil {
+			err = tx.Commit()
+		} else {
+			tx.Rollback()
+		}
+	}()
+
+	res, err := tx.Stmt(s.prep["cleanAddress"]).Exec()
+	if err != nil {
+		return err
+	}
+	if rows, _ := res.RowsAffected(); rows > 0 {
+		log.Printf("Clean: %d old addresses", rows)
+	}
+
+	res, err = tx.Stmt(s.prep["cleanDevice"]).Exec()
+	if err != nil {
+		return err
+	}
+	if rows, _ := res.RowsAffected(); rows > 0 {
+		log.Printf("Clean: %d old devices", rows)
+	}
+
+	var devs, addrs int
+	row := tx.Stmt(s.prep["countDevice"]).QueryRow()
+	if err = row.Scan(&devs); err != nil {
+		return err
+	}
+	row = tx.Stmt(s.prep["countAddress"]).QueryRow()
+	if err = row.Scan(&addrs); err != nil {
+		return err
+	}
+
+	log.Printf("Database: %d devices, %d addresses", devs, addrs)
+	return nil
+}

cmd/stdiscosrv/database.go

@@ -1,383 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//go:generate go run ../../proto/scripts/protofmt.go database.proto
-//go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
-
-package main
-
-import (
-	"context"
-	"log"
-	"sort"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/sliceutil"
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/storage"
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-type clock interface {
-	Now() time.Time
-}
-
-type defaultClock struct{}
-
-func (defaultClock) Now() time.Time {
-	return time.Now()
-}
-
-type database interface {
-	put(key string, rec DatabaseRecord) error
-	merge(key string, addrs []DatabaseAddress, seen int64) error
-	get(key string) (DatabaseRecord, error)
-}
-
-type levelDBStore struct {
-	db         *leveldb.DB
-	inbox      chan func()
-	clock      clock
-	marshalBuf []byte
-}
-
-func newLevelDBStore(dir string) (*levelDBStore, error) {
-	db, err := leveldb.OpenFile(dir, levelDBOptions)
-	if err != nil {
-		return nil, err
-	}
-	return &levelDBStore{
-		db:    db,
-		inbox: make(chan func(), 16),
-		clock: defaultClock{},
-	}, nil
-}
-
-func newMemoryLevelDBStore() (*levelDBStore, error) {
-	db, err := leveldb.Open(storage.NewMemStorage(), nil)
-	if err != nil {
-		return nil, err
-	}
-	return &levelDBStore{
-		db:    db,
-		inbox: make(chan func(), 16),
-		clock: defaultClock{},
-	}, nil
-}
-
-func (s *levelDBStore) put(key string, rec DatabaseRecord) error {
-	t0 := time.Now()
-	defer func() {
-		databaseOperationSeconds.WithLabelValues(dbOpPut).Observe(time.Since(t0).Seconds())
-	}()
-
-	rc := make(chan error)
-
-	s.inbox <- func() {
-		size := rec.Size()
-		if len(s.marshalBuf) < size {
-			s.marshalBuf = make([]byte, size)
-		}
-		n, _ := rec.MarshalTo(s.marshalBuf)
-		rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
-	}
-
-	err := <-rc
-	if err != nil {
-		databaseOperations.WithLabelValues(dbOpPut, dbResError).Inc()
-	} else {
-		databaseOperations.WithLabelValues(dbOpPut, dbResSuccess).Inc()
-	}
-
-	return err
-}
-
-func (s *levelDBStore) merge(key string, addrs []DatabaseAddress, seen int64) error {
-	t0 := time.Now()
-	defer func() {
-		databaseOperationSeconds.WithLabelValues(dbOpMerge).Observe(time.Since(t0).Seconds())
-	}()
-
-	rc := make(chan error)
-	newRec := DatabaseRecord{
-		Addresses: addrs,
-		Seen:      seen,
-	}
-
-	s.inbox <- func() {
-		// grab the existing record
-		oldRec, err := s.get(key)
-		if err != nil {
-			// "not found" is not an error from get, so this is serious
-			// stuff only
-			rc <- err
-			return
-		}
-		newRec = merge(newRec, oldRec)
-
-		// We replicate s.put() functionality here ourselves instead of
-		// calling it because we want to serialize our get above together
-		// with the put in the same function.
-		size := newRec.Size()
-		if len(s.marshalBuf) < size {
-			s.marshalBuf = make([]byte, size)
-		}
-		n, _ := newRec.MarshalTo(s.marshalBuf)
-		rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
-	}
-
-	err := <-rc
-	if err != nil {
-		databaseOperations.WithLabelValues(dbOpMerge, dbResError).Inc()
-	} else {
-		databaseOperations.WithLabelValues(dbOpMerge, dbResSuccess).Inc()
-	}
-
-	return err
-}
-
-func (s *levelDBStore) get(key string) (DatabaseRecord, error) {
-	t0 := time.Now()
-	defer func() {
-		databaseOperationSeconds.WithLabelValues(dbOpGet).Observe(time.Since(t0).Seconds())
-	}()
-
-	keyBs := []byte(key)
-	val, err := s.db.Get(keyBs, nil)
-	if err == leveldb.ErrNotFound {
-		databaseOperations.WithLabelValues(dbOpGet, dbResNotFound).Inc()
-		return DatabaseRecord{}, nil
-	}
-	if err != nil {
-		databaseOperations.WithLabelValues(dbOpGet, dbResError).Inc()
-		return DatabaseRecord{}, err
-	}
-
-	var rec DatabaseRecord
-
-	if err := rec.Unmarshal(val); err != nil {
-		databaseOperations.WithLabelValues(dbOpGet, dbResUnmarshalError).Inc()
-		return DatabaseRecord{}, nil
-	}
-
-	rec.Addresses = expire(rec.Addresses, s.clock.Now().UnixNano())
-	databaseOperations.WithLabelValues(dbOpGet, dbResSuccess).Inc()
-	return rec, nil
-}
-
-func (s *levelDBStore) Serve(ctx context.Context) error {
-	t := time.NewTimer(0)
-	defer t.Stop()
-	defer s.db.Close()
-
-	// Start the statistics serve routine. It will exit with us when
-	// statisticsTrigger is closed.
-	statisticsTrigger := make(chan struct{})
-	statisticsDone := make(chan struct{})
-	go s.statisticsServe(statisticsTrigger, statisticsDone)
-
-loop:
-	for {
-		select {
-		case fn := <-s.inbox:
-			// Run function in serialized order.
-			fn()
-
-		case <-t.C:
-			// Trigger the statistics routine to do its thing in the
-			// background.
-			statisticsTrigger <- struct{}{}
-
-		case <-statisticsDone:
-			// The statistics routine is done with one iteratation, schedule
-			// the next.
-			t.Reset(databaseStatisticsInterval)
-
-		case <-ctx.Done():
-			// We're done.
-			close(statisticsTrigger)
-			break loop
-		}
-	}
-
-	// Also wait for statisticsServe to return
-	<-statisticsDone
-
-	return nil
-}
-
-func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
-	defer close(done)
-
-	for range trigger {
-		t0 := time.Now()
-		nowNanos := t0.UnixNano()
-		cutoff24h := t0.Add(-24 * time.Hour).UnixNano()
-		cutoff1w := t0.Add(-7 * 24 * time.Hour).UnixNano()
-		cutoff2Mon := t0.Add(-60 * 24 * time.Hour).UnixNano()
-		current, last24h, last1w, inactive, errors := 0, 0, 0, 0, 0
-
-		iter := s.db.NewIterator(&util.Range{}, nil)
-		for iter.Next() {
-			// Attempt to unmarshal the record and count the
-			// failure if there's something wrong with it.
-			var rec DatabaseRecord
-			if err := rec.Unmarshal(iter.Value()); err != nil {
-				errors++
-				continue
-			}
-
-			// If there are addresses that have not expired it's a current
-			// record, otherwise account it based on when it was last seen
-			// (last 24 hours or last week) or finally as inactice.
-			switch {
-			case len(expire(rec.Addresses, nowNanos)) > 0:
-				current++
-			case rec.Seen > cutoff24h:
-				last24h++
-			case rec.Seen > cutoff1w:
-				last1w++
-			case rec.Seen > cutoff2Mon:
-				inactive++
-			case rec.Missed < cutoff2Mon:
-				// It hasn't been seen lately and we haven't recorded
-				// someone asking for this device in a long time either;
-				// delete the record.
-				if err := s.db.Delete(iter.Key(), nil); err != nil {
-					databaseOperations.WithLabelValues(dbOpDelete, dbResError).Inc()
-				} else {
-					databaseOperations.WithLabelValues(dbOpDelete, dbResSuccess).Inc()
-				}
-			default:
-				inactive++
-			}
-		}
-
-		iter.Release()
-
-		databaseKeys.WithLabelValues("current").Set(float64(current))
-		databaseKeys.WithLabelValues("last24h").Set(float64(last24h))
-		databaseKeys.WithLabelValues("last1w").Set(float64(last1w))
-		databaseKeys.WithLabelValues("inactive").Set(float64(inactive))
-		databaseKeys.WithLabelValues("error").Set(float64(errors))
-		databaseStatisticsSeconds.Set(time.Since(t0).Seconds())
-
-		// Signal that we are done and can be scheduled again.
-		done <- struct{}{}
-	}
-}
-
-// merge returns the merged result of the two database records a and b. The
-// result is the union of the two address sets, with the newer expiry time
-// chosen for any duplicates.
-func merge(a, b DatabaseRecord) DatabaseRecord {
-	// Both lists must be sorted for this to work.
-	if !sort.IsSorted(databaseAddressOrder(a.Addresses)) {
-		log.Println("Warning: bug: addresses not correctly sorted in merge")
-		a.Addresses = sortedAddressCopy(a.Addresses)
-	}
-	if !sort.IsSorted(databaseAddressOrder(b.Addresses)) {
-		// no warning because this is the side we read from disk and it may
-		// legitimately predate correct sorting.
-		b.Addresses = sortedAddressCopy(b.Addresses)
-	}
-
-	res := DatabaseRecord{
-		Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
-		Seen:      a.Seen,
-	}
-	if b.Seen > a.Seen {
-		res.Seen = b.Seen
-	}
-
-	aIdx := 0
-	bIdx := 0
-	aAddrs := a.Addresses
-	bAddrs := b.Addresses
-loop:
-	for {
-		switch {
-		case aIdx == len(aAddrs) && bIdx == len(bAddrs):
-			// both lists are exhausted, we are done
-			break loop
-
-		case aIdx == len(aAddrs):
-			// a is exhausted, pick from b and continue
-			res.Addresses = append(res.Addresses, bAddrs[bIdx])
-			bIdx++
-			continue
-
-		case bIdx == len(bAddrs):
-			// b is exhausted, pick from a and continue
-			res.Addresses = append(res.Addresses, aAddrs[aIdx])
-			aIdx++
-			continue
-		}
-
-		// We have values left on both sides.
-		aVal := aAddrs[aIdx]
-		bVal := bAddrs[bIdx]
-
-		switch {
-		case aVal.Address == bVal.Address:
-			// update for same address, pick newer
-			if aVal.Expires > bVal.Expires {
-				res.Addresses = append(res.Addresses, aVal)
-			} else {
-				res.Addresses = append(res.Addresses, bVal)
-			}
-			aIdx++
-			bIdx++
-
-		case aVal.Address < bVal.Address:
-			// a is smallest, pick it and continue
-			res.Addresses = append(res.Addresses, aVal)
-			aIdx++
-
-		default:
-			// b is smallest, pick it and continue
-			res.Addresses = append(res.Addresses, bVal)
-			bIdx++
-		}
-	}
-	return res
-}
-
-// expire returns the list of addresses after removing expired entries.
-// Expiration happen in place, so the slice given as the parameter is
-// destroyed. Internal order is not preserved.
-func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
-	i := 0
-	for i < len(addrs) {
-		if addrs[i].Expires < now {
-			addrs = sliceutil.RemoveAndZero(addrs, i)
-			continue
-		}
-		i++
-	}
-	return addrs
-}
-
-func sortedAddressCopy(addrs []DatabaseAddress) []DatabaseAddress {
-	sorted := make([]DatabaseAddress, len(addrs))
-	copy(sorted, addrs)
-	sort.Sort(databaseAddressOrder(sorted))
-	return sorted
-}
-
-type databaseAddressOrder []DatabaseAddress
-
-func (s databaseAddressOrder) Less(a, b int) bool {
-	return s[a].Address < s[b].Address
-}
-
-func (s databaseAddressOrder) Swap(a, b int) {
-	s[a], s[b] = s[b], s[a]
-}
-
-func (s databaseAddressOrder) Len() int {
-	return len(s)
-}

cmd/stdiscosrv/database.pb.go

@@ -1,847 +0,0 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: database.proto
-
-package main
-
-import (
-	fmt "fmt"
-	_ "github.com/gogo/protobuf/gogoproto"
-	proto "github.com/gogo/protobuf/proto"
-	io "io"
-	math "math"
-	math_bits "math/bits"
-)
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
-
-type DatabaseRecord struct {
-	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
-	Misses    int32             `protobuf:"varint,2,opt,name=misses,proto3" json:"misses,omitempty"`
-	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
-	Missed    int64             `protobuf:"varint,4,opt,name=missed,proto3" json:"missed,omitempty"`
-}
-
-func (m *DatabaseRecord) Reset()         { *m = DatabaseRecord{} }
-func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
-func (*DatabaseRecord) ProtoMessage()    {}
-func (*DatabaseRecord) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b90fe3356ea5df07, []int{0}
-}
-func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
-	return m.Unmarshal(b)
-}
-func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	if deterministic {
-		return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
-	} else {
-		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
-		if err != nil {
-			return nil, err
-		}
-		return b[:n], nil
-	}
-}
-func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DatabaseRecord.Merge(m, src)
-}
-func (m *DatabaseRecord) XXX_Size() int {
-	return m.Size()
-}
-func (m *DatabaseRecord) XXX_DiscardUnknown() {
-	xxx_messageInfo_DatabaseRecord.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_DatabaseRecord proto.InternalMessageInfo
-
-type ReplicationRecord struct {
-	Key       string            `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
-	Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses"`
-	Seen      int64             `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
-}
-
-func (m *ReplicationRecord) Reset()         { *m = ReplicationRecord{} }
-func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
-func (*ReplicationRecord) ProtoMessage()    {}
-func (*ReplicationRecord) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b90fe3356ea5df07, []int{1}
-}
-func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
-	return m.Unmarshal(b)
-}
-func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	if deterministic {
-		return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
-	} else {
-		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
-		if err != nil {
-			return nil, err
-		}
-		return b[:n], nil
-	}
-}
-func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ReplicationRecord.Merge(m, src)
-}
-func (m *ReplicationRecord) XXX_Size() int {
-	return m.Size()
-}
-func (m *ReplicationRecord) XXX_DiscardUnknown() {
-	xxx_messageInfo_ReplicationRecord.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_ReplicationRecord proto.InternalMessageInfo
-
-type DatabaseAddress struct {
-	Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
-	Expires int64  `protobuf:"varint,2,opt,name=expires,proto3" json:"expires,omitempty"`
-}
-
-func (m *DatabaseAddress) Reset()         { *m = DatabaseAddress{} }
-func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
-func (*DatabaseAddress) ProtoMessage()    {}
-func (*DatabaseAddress) Descriptor() ([]byte, []int) {
-	return fileDescriptor_b90fe3356ea5df07, []int{2}
-}
-func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
-	return m.Unmarshal(b)
-}
-func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	if deterministic {
-		return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
-	} else {
-		b = b[:cap(b)]
-		n, err := m.MarshalToSizedBuffer(b)
-		if err != nil {
-			return nil, err
-		}
-		return b[:n], nil
-	}
-}
-func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DatabaseAddress.Merge(m, src)
-}
-func (m *DatabaseAddress) XXX_Size() int {
-	return m.Size()
-}
-func (m *DatabaseAddress) XXX_DiscardUnknown() {
-	xxx_messageInfo_DatabaseAddress.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_DatabaseAddress proto.InternalMessageInfo
-
-func init() {
-	proto.RegisterType((*DatabaseRecord)(nil), "main.DatabaseRecord")
-	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
-	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
-}
-
-func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
-
-var fileDescriptor_b90fe3356ea5df07 = []byte{
-	// 270 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
-	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
-	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
-	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
-	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
-	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
-	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
-	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
-	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
-	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
-	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
-	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
-	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
-	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
-	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
-	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
-	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
-}
-
-func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if m.Missed != 0 {
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
-		i--
-		dAtA[i] = 0x20
-	}
-	if m.Seen != 0 {
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
-		i--
-		dAtA[i] = 0x18
-	}
-	if m.Misses != 0 {
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
-		i--
-		dAtA[i] = 0x10
-	}
-	if len(m.Addresses) > 0 {
-		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
-			{
-				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-				if err != nil {
-					return 0, err
-				}
-				i -= size
-				i = encodeVarintDatabase(dAtA, i, uint64(size))
-			}
-			i--
-			dAtA[i] = 0xa
-		}
-	}
-	return len(dAtA) - i, nil
-}
-
-func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if m.Seen != 0 {
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
-		i--
-		dAtA[i] = 0x18
-	}
-	if len(m.Addresses) > 0 {
-		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
-			{
-				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-				if err != nil {
-					return 0, err
-				}
-				i -= size
-				i = encodeVarintDatabase(dAtA, i, uint64(size))
-			}
-			i--
-			dAtA[i] = 0x12
-		}
-	}
-	if len(m.Key) > 0 {
-		i -= len(m.Key)
-		copy(dAtA[i:], m.Key)
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
-		i--
-		dAtA[i] = 0xa
-	}
-	return len(dAtA) - i, nil
-}
-
-func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if m.Expires != 0 {
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
-		i--
-		dAtA[i] = 0x10
-	}
-	if len(m.Address) > 0 {
-		i -= len(m.Address)
-		copy(dAtA[i:], m.Address)
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
-		i--
-		dAtA[i] = 0xa
-	}
-	return len(dAtA) - i, nil
-}
-
-func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
-	offset -= sovDatabase(v)
-	base := offset
-	for v >= 1<<7 {
-		dAtA[offset] = uint8(v&0x7f | 0x80)
-		v >>= 7
-		offset++
-	}
-	dAtA[offset] = uint8(v)
-	return base
-}
-func (m *DatabaseRecord) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	if len(m.Addresses) > 0 {
-		for _, e := range m.Addresses {
-			l = e.Size()
-			n += 1 + l + sovDatabase(uint64(l))
-		}
-	}
-	if m.Misses != 0 {
-		n += 1 + sovDatabase(uint64(m.Misses))
-	}
-	if m.Seen != 0 {
-		n += 1 + sovDatabase(uint64(m.Seen))
-	}
-	if m.Missed != 0 {
-		n += 1 + sovDatabase(uint64(m.Missed))
-	}
-	return n
-}
-
-func (m *ReplicationRecord) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	l = len(m.Key)
-	if l > 0 {
-		n += 1 + l + sovDatabase(uint64(l))
-	}
-	if len(m.Addresses) > 0 {
-		for _, e := range m.Addresses {
-			l = e.Size()
-			n += 1 + l + sovDatabase(uint64(l))
-		}
-	}
-	if m.Seen != 0 {
-		n += 1 + sovDatabase(uint64(m.Seen))
-	}
-	return n
-}
-
-func (m *DatabaseAddress) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	l = len(m.Address)
-	if l > 0 {
-		n += 1 + l + sovDatabase(uint64(l))
-	}
-	if m.Expires != 0 {
-		n += 1 + sovDatabase(uint64(m.Expires))
-	}
-	return n
-}
-
-func sovDatabase(x uint64) (n int) {
-	return (math_bits.Len64(x|1) + 6) / 7
-}
-func sozDatabase(x uint64) (n int) {
-	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
-}
-func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowDatabase
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: DatabaseRecord: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: DatabaseRecord: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Addresses = append(m.Addresses, DatabaseAddress{})
-			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		case 2:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Misses", wireType)
-			}
-			m.Misses = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Misses |= int32(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		case 3:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
-			}
-			m.Seen = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Seen |= int64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		case 4:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Missed", wireType)
-			}
-			m.Missed = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Missed |= int64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		default:
-			iNdEx = preIndex
-			skippy, err := skipDatabase(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if (skippy < 0) || (iNdEx+skippy) < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowDatabase
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: ReplicationRecord: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: ReplicationRecord: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Key = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Addresses = append(m.Addresses, DatabaseAddress{})
-			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		case 3:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
-			}
-			m.Seen = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Seen |= int64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		default:
-			iNdEx = preIndex
-			skippy, err := skipDatabase(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if (skippy < 0) || (iNdEx+skippy) < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowDatabase
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: DatabaseAddress: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: DatabaseAddress: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Address", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Address = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		case 2:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Expires", wireType)
-			}
-			m.Expires = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Expires |= int64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		default:
-			iNdEx = preIndex
-			skippy, err := skipDatabase(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if (skippy < 0) || (iNdEx+skippy) < 0 {
-				return ErrInvalidLengthDatabase
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-func skipDatabase(dAtA []byte) (n int, err error) {
-	l := len(dAtA)
-	iNdEx := 0
-	depth := 0
-	for iNdEx < l {
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return 0, ErrIntOverflowDatabase
-			}
-			if iNdEx >= l {
-				return 0, io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		wireType := int(wire & 0x7)
-		switch wireType {
-		case 0:
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return 0, ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return 0, io.ErrUnexpectedEOF
-				}
-				iNdEx++
-				if dAtA[iNdEx-1] < 0x80 {
-					break
-				}
-			}
-		case 1:
-			iNdEx += 8
-		case 2:
-			var length int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return 0, ErrIntOverflowDatabase
-				}
-				if iNdEx >= l {
-					return 0, io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				length |= (int(b) & 0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if length < 0 {
-				return 0, ErrInvalidLengthDatabase
-			}
-			iNdEx += length
-		case 3:
-			depth++
-		case 4:
-			if depth == 0 {
-				return 0, ErrUnexpectedEndOfGroupDatabase
-			}
-			depth--
-		case 5:
-			iNdEx += 4
-		default:
-			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
-		}
-		if iNdEx < 0 {
-			return 0, ErrInvalidLengthDatabase
-		}
-		if depth == 0 {
-			return iNdEx, nil
-		}
-	}
-	return 0, io.ErrUnexpectedEOF
-}
-
-var (
-	ErrInvalidLengthDatabase        = fmt.Errorf("proto: negative length found during unmarshaling")
-	ErrIntOverflowDatabase          = fmt.Errorf("proto: integer overflow")
-	ErrUnexpectedEndOfGroupDatabase = fmt.Errorf("proto: unexpected end of group")
-)

cmd/stdiscosrv/database.proto

@@ -1,36 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-syntax = "proto3";
-
-package main;
-
-import "repos/protobuf/gogoproto/gogo.proto";
-
-option (gogoproto.goproto_getters_all) = false;
-option (gogoproto.goproto_unkeyed_all) = false;
-option (gogoproto.goproto_unrecognized_all) = false;
-option (gogoproto.goproto_sizecache_all) = false;
-
-message DatabaseRecord {
-    repeated DatabaseAddress addresses = 1 [(gogoproto.nullable) = false];
-    int32                    misses    = 2; // Number of lookups* without hits
-    int64                    seen      = 3; // Unix nanos, last device announce
-    int64                    missed    = 4; // Unix nanos, last* failed lookup
-}
-
-// *) Not every lookup results in a write, so may not be completely accurate
-
-message ReplicationRecord {
-    string                   key       = 1;
-    repeated DatabaseAddress addresses = 2 [(gogoproto.nullable) = false];
-    int64                    seen      = 3; // Unix nanos, last device announce
-}
-
-message DatabaseAddress {
-    string address = 1;
-    int64  expires = 2; // Unix nanos
-}

cmd/stdiscosrv/database_test.go

@@ -1,211 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-)
-
-func TestDatabaseGetSet(t *testing.T) {
-	db, err := newMemoryLevelDBStore()
-	if err != nil {
-		t.Fatal(err)
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	go db.Serve(ctx)
-	defer cancel()
-
-	// Check missing record
-
-	rec, err := db.get("abcd")
-	if err != nil {
-		t.Error("not found should not be an error")
-	}
-	if len(rec.Addresses) != 0 {
-		t.Error("addresses should be empty")
-	}
-	if rec.Misses != 0 {
-		t.Error("missing should be zero")
-	}
-
-	// Set up a clock
-
-	now := time.Now()
-	tc := &testClock{now}
-	db.clock = tc
-
-	// Put a record
-
-	rec.Addresses = []DatabaseAddress{
-		{Address: "tcp://1.2.3.4:5", Expires: tc.Now().Add(time.Minute).UnixNano()},
-	}
-	if err := db.put("abcd", rec); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify it
-
-	rec, err = db.get("abcd")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(rec.Addresses) != 1 {
-		t.Log(rec.Addresses)
-		t.Fatal("should have one address")
-	}
-	if rec.Addresses[0].Address != "tcp://1.2.3.4:5" {
-		t.Log(rec.Addresses)
-		t.Error("incorrect address")
-	}
-
-	// Wind the clock one half expiry, and merge in a new address
-
-	tc.wind(30 * time.Second)
-
-	addrs := []DatabaseAddress{
-		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
-	}
-	if err := db.merge("abcd", addrs, tc.Now().UnixNano()); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify it
-
-	rec, err = db.get("abcd")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(rec.Addresses) != 2 {
-		t.Log(rec.Addresses)
-		t.Fatal("should have two addresses")
-	}
-	if rec.Addresses[0].Address != "tcp://1.2.3.4:5" {
-		t.Log(rec.Addresses)
-		t.Error("incorrect address[0]")
-	}
-	if rec.Addresses[1].Address != "tcp://6.7.8.9:0" {
-		t.Log(rec.Addresses)
-		t.Error("incorrect address[1]")
-	}
-
-	// Pass the first expiry time
-
-	tc.wind(45 * time.Second)
-
-	// Verify it
-
-	rec, err = db.get("abcd")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(rec.Addresses) != 1 {
-		t.Log(rec.Addresses)
-		t.Fatal("should have one address")
-	}
-	if rec.Addresses[0].Address != "tcp://6.7.8.9:0" {
-		t.Log(rec.Addresses)
-		t.Error("incorrect address")
-	}
-
-	// Put a record with misses
-
-	rec = DatabaseRecord{Misses: 42, Missed: tc.Now().UnixNano()}
-	if err := db.put("efgh", rec); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify it
-
-	rec, err = db.get("efgh")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(rec.Addresses) != 0 {
-		t.Log(rec.Addresses)
-		t.Fatal("should have no addresses")
-	}
-	if rec.Misses != 42 {
-		t.Log(rec.Misses)
-		t.Error("incorrect misses")
-	}
-
-	// Set an address
-
-	addrs = []DatabaseAddress{
-		{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
-	}
-	if err := db.merge("efgh", addrs, tc.Now().UnixNano()); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify it
-
-	rec, err = db.get("efgh")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(rec.Addresses) != 1 {
-		t.Log(rec.Addresses)
-		t.Fatal("should have one address")
-	}
-	if rec.Misses != 0 {
-		t.Log(rec.Misses)
-		t.Error("should have no misses")
-	}
-}
-
-func TestFilter(t *testing.T) {
-	// all cases are expired with t=10
-	cases := []struct {
-		a []DatabaseAddress
-		b []DatabaseAddress
-	}{
-		{
-			a: nil,
-			b: nil,
-		},
-		{
-			a: []DatabaseAddress{{Address: "a", Expires: 9}, {Address: "b", Expires: 9}, {Address: "c", Expires: 9}},
-			b: []DatabaseAddress{},
-		},
-		{
-			a: []DatabaseAddress{{Address: "a", Expires: 10}},
-			b: []DatabaseAddress{{Address: "a", Expires: 10}},
-		},
-		{
-			a: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
-			b: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
-		},
-		{
-			a: []DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
-			b: []DatabaseAddress{{Address: "b", Expires: 15}, {Address: "d", Expires: 15}},
-		},
-	}
-
-	for _, tc := range cases {
-		res := expire(tc.a, 10)
-		if fmt.Sprint(res) != fmt.Sprint(tc.b) {
-			t.Errorf("Incorrect result %v, expected %v", res, tc.b)
-		}
-	}
-}
-
-type testClock struct {
-	now time.Time
-}
-
-func (t *testClock) wind(d time.Duration) {
-	t.now = t.now.Add(d)
-}
-
-func (t *testClock) Now() time.Time {
-	t.now = t.now.Add(time.Nanosecond)
-	return t.now
-}

cmd/stdiscosrv/db.go

@@ -0,0 +1,32 @@
+// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+type setupFunc func(db *sql.DB) error
+type compileFunc func(db *sql.DB) (map[string]*sql.Stmt, error)
+
+var (
+	setupFuncs   = make(map[string]setupFunc)
+	compileFuncs = make(map[string]compileFunc)
+)
+
+func register(name string, setup setupFunc, compile compileFunc) {
+	setupFuncs[name] = setup
+	compileFuncs[name] = compile
+}
+
+func setup(backend string, db *sql.DB) (map[string]*sql.Stmt, error) {
+	setup, ok := setupFuncs[backend]
+	if !ok {
+		return nil, fmt.Errorf("Unsupported backend")
+	}
+	if err := setup(db); err != nil {
+		return nil, err
+	}
+	return compileFuncs[backend](db)
+}

cmd/stdiscosrv/etc/firewall-ufw/stdiscosrv

@@ -1,4 +0,0 @@
-[stdiscosrv]
-title=Syncthing discovery server
-description=Lets syncthing clients discover each other
-ports=8443/tcp

cmd/stdiscosrv/etc/linux-systemd/default

@@ -1,3 +0,0 @@
-# Default settings for syncthing-discosrv (stdiscosrv).
-## Add Options here:
-DISCOSRV_OPTS=

cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service

@@ -1,25 +0,0 @@
-[Unit]
-Description=Syncthing Discovery Server
-After=network.target
-Documentation=man:stdiscosrv(1)
-
-[Service]
-WorkingDirectory=/var/lib/syncthing-discosrv
-EnvironmentFile=/etc/default/syncthing-discosrv
-ExecStart=/usr/bin/stdiscosrv $DISCOSRV_OPTS
-
-# Hardening
-User=syncthing-discosrv
-Group=syncthing
-ProtectSystem=strict
-ReadWritePaths=/var/lib/syncthing-discosrv
-NoNewPrivileges=true
-PrivateTmp=true
-PrivateDevices=true
-ProtectHome=true
-SystemCallArchitectures=native
-MemoryDenyWriteExecute=true
-
-[Install]
-WantedBy=multi-user.target
-Alias=syncthing-discosrv.service

cmd/stdiscosrv/main.go

@@ -1,217 +1,146 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
+// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
 
 package main
 
 import (
-	"context"
 	"crypto/tls"
+	"database/sql"
 	"flag"
+	"fmt"
 	"log"
-	"net"
-	"net/http"
 	"os"
-	"strings"
+	"runtime"
+	"strconv"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/thejerf/suture/v4"
+	"github.com/thejerf/suture"
 )
 
 const (
-	addressExpiryTime          = 2 * time.Hour
-	databaseStatisticsInterval = 5 * time.Minute
-
-	// Reannounce-After is set to reannounceAfterSeconds +
-	// random(reannounzeFuzzSeconds), similar for Retry-After
-	reannounceAfterSeconds = 3300
-	reannounzeFuzzSeconds  = 300
-	errorRetryAfterSeconds = 1500
-	errorRetryFuzzSeconds  = 300
-
-	// Retry for not found is minSeconds + failures * incSeconds +
-	// random(fuzz), where failures is the number of consecutive lookups
-	// with no answer, up to maxSeconds. The fuzz is applied after capping
-	// to maxSeconds.
-	notFoundRetryMinSeconds  = 60
-	notFoundRetryMaxSeconds  = 3540
-	notFoundRetryIncSeconds  = 10
-	notFoundRetryFuzzSeconds = 60
-
-	// How often (in requests) we serialize the missed counter to database.
-	notFoundMissesWriteInterval = 10
-
-	httpReadTimeout    = 5 * time.Second
-	httpWriteTimeout   = 5 * time.Second
-	httpMaxHeaderBytes = 1 << 10
-
-	// Size of the replication outbox channel
-	replicationOutboxSize = 10000
+	minNegCache  = 60        // seconds
+	maxNegCache  = 3600      // seconds
+	maxDeviceAge = 7 * 86400 // one week, in seconds
 )
 
-// These options make the database a little more optimized for writes, at
-// the expense of some memory usage and risk of losing writes in a (system)
-// crash.
-var levelDBOptions = &opt.Options{
-	NoSync:      true,
-	WriteBuffer: 32 << 20, // default 4<<20
+var (
+	Version    string
+	BuildStamp string
+	BuildUser  string
+	BuildHost  string
+
+	BuildDate   time.Time
+	LongVersion string
+)
+
+func init() {
+	stamp, _ := strconv.Atoi(BuildStamp)
+	BuildDate = time.Unix(int64(stamp), 0)
+
+	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
+	LongVersion = fmt.Sprintf(`stdiscosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
 }
 
-var debug = false
+var (
+	lruSize     = 10240
+	limitAvg    = 5
+	limitBurst  = 20
+	globalStats stats
+	statsFile   string
+	backend     = "ql"
+	dsn         = getEnvDefault("STDISCOSRV_DB_DSN", "memory://stdiscosrv")
+	certFile    = "cert.pem"
+	keyFile     = "key.pem"
+	debug       = false
+	useHTTP     = false
+)
 
 func main() {
+	const (
+		cleanIntv = 1 * time.Hour
+		statsIntv = 5 * time.Minute
+	)
+
 	var listen string
-	var dir string
-	var metricsListen string
-	var replicationListen string
-	var replicationPeers string
-	var certFile string
-	var keyFile string
-	var replCertFile string
-	var replKeyFile string
-	var useHTTP bool
-	var largeDB bool
 
 	log.SetOutput(os.Stdout)
 	log.SetFlags(0)
 
-	flag.StringVar(&certFile, "cert", "./cert.pem", "Certificate file")
-	flag.StringVar(&keyFile, "key", "./key.pem", "Key file")
-	flag.StringVar(&dir, "db-dir", "./discovery.db", "Database directory")
-	flag.BoolVar(&debug, "debug", false, "Print debug output")
-	flag.BoolVar(&useHTTP, "http", false, "Listen on HTTP (behind an HTTPS proxy)")
 	flag.StringVar(&listen, "listen", ":8443", "Listen address")
-	flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
-	flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
-	flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
-	flag.StringVar(&replCertFile, "replication-cert", "", "Certificate file for replication")
-	flag.StringVar(&replKeyFile, "replication-key", "", "Key file for replication")
-	flag.BoolVar(&largeDB, "large-db", false, "Use larger database settings")
-	showVersion := flag.Bool("version", false, "Show version")
+	flag.IntVar(&lruSize, "limit-cache", lruSize, "Limiter cache entries")
+	flag.IntVar(&limitAvg, "limit-avg", limitAvg, "Allowed average package rate, per 10 s")
+	flag.IntVar(&limitBurst, "limit-burst", limitBurst, "Allowed burst size, packets")
+	flag.StringVar(&statsFile, "stats-file", statsFile, "File to write periodic operation stats to")
+	flag.StringVar(&backend, "db-backend", backend, "Database backend to use")
+	flag.StringVar(&dsn, "db-dsn", dsn, "Database DSN")
+	flag.StringVar(&certFile, "cert", certFile, "Certificate file")
+	flag.StringVar(&keyFile, "key", keyFile, "Key file")
+	flag.BoolVar(&debug, "debug", debug, "Debug")
+	flag.BoolVar(&useHTTP, "http", useHTTP, "Listen on HTTP (behind an HTTPS proxy)")
 	flag.Parse()
 
-	log.Println(build.LongVersionFor("stdiscosrv"))
-	if *showVersion {
-		return
-	}
+	log.Println(LongVersion)
 
-	if largeDB {
-		levelDBOptions.BlockCacheCapacity = 64 << 20
-		levelDBOptions.BlockSize = 64 << 10
-		levelDBOptions.CompactionTableSize = 16 << 20
-		levelDBOptions.CompactionTableSizeMultiplier = 2.0
-		levelDBOptions.WriteBuffer = 64 << 20
-		levelDBOptions.CompactionL0Trigger = 8
-	}
-
-	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
-	if os.IsNotExist(err) {
-		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 20*365)
+	var cert tls.Certificate
+	var err error
+	if !useHTTP {
+		cert, err = tls.LoadX509KeyPair(certFile, keyFile)
 		if err != nil {
-			log.Fatalln("Failed to generate X509 key pair:", err)
-		}
-	} else if err != nil {
-		log.Fatalln("Failed to load keypair:", err)
-	}
-	devID := protocol.NewDeviceID(cert.Certificate[0])
-	log.Println("Server device ID is", devID)
-
-	replCert := cert
-	if replCertFile != "" && replKeyFile != "" {
-		replCert, err = tls.LoadX509KeyPair(replCertFile, replKeyFile)
-		if err != nil {
-			log.Fatalln("Failed to load replication keypair:", err)
-		}
-	}
-	replDevID := protocol.NewDeviceID(replCert.Certificate[0])
-	log.Println("Replication device ID is", replDevID)
-
-	// Parse the replication specs, if any.
-	var allowedReplicationPeers []protocol.DeviceID
-	var replicationDestinations []string
-	parts := strings.Split(replicationPeers, ",")
-	for _, part := range parts {
-		if part == "" {
-			continue
-		}
-
-		fields := strings.Split(part, "@")
-		switch len(fields) {
-		case 2:
-			// This is an id@address specification. Grab the address for the
-			// destination list. Try to resolve it once to catch obvious
-			// syntax errors here rather than having the sender service fail
-			// repeatedly later.
-			_, err := net.ResolveTCPAddr("tcp", fields[1])
+			log.Println("Failed to load keypair. Generating one, this might take a while...")
+			cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 3072)
 			if err != nil {
-				log.Fatalln("Resolving address:", err)
+				log.Fatalln("Failed to generate X509 key pair:", err)
 			}
-			replicationDestinations = append(replicationDestinations, fields[1])
-			fallthrough // N.B.
-
-		case 1:
-			// The first part is always a device ID.
-			id, err := protocol.DeviceIDFromString(fields[0])
-			if err != nil {
-				log.Fatalln("Parsing device ID:", err)
-			}
-			if id == protocol.EmptyDeviceID {
-				log.Fatalf("Missing device ID for peer in %q", part)
-			}
-			allowedReplicationPeers = append(allowedReplicationPeers, id)
-
-		default:
-			log.Fatalln("Unrecognized replication spec:", part)
 		}
+
+		devID := protocol.NewDeviceID(cert.Certificate[0])
+		log.Println("Server device ID is", devID)
 	}
 
-	// Root of the service tree.
-	main := suture.New("main", suture.Spec{
-		PassThroughPanics: true,
+	db, err := sql.Open(backend, dsn)
+	if err != nil {
+		log.Fatalln("sql.Open:", err)
+	}
+	prep, err := setup(backend, db)
+	if err != nil {
+		log.Fatalln("Setup:", err)
+	}
+
+	main := suture.NewSimple("main")
+
+	main.Add(&querysrv{
+		addr: listen,
+		cert: cert,
+		db:   db,
+		prep: prep,
 	})
 
-	// Start the database.
-	db, err := newLevelDBStore(dir)
-	if err != nil {
-		log.Fatalln("Open database:", err)
-	}
-	main.Add(db)
+	main.Add(&cleansrv{
+		intv: cleanIntv,
+		db:   db,
+		prep: prep,
+	})
 
-	// Start any replication senders.
-	var repl replicationMultiplexer
-	for _, dst := range replicationDestinations {
-		rs := newReplicationSender(dst, replCert, allowedReplicationPeers)
-		main.Add(rs)
-		repl = append(repl, rs)
-	}
+	main.Add(&statssrv{
+		intv: statsIntv,
+		file: statsFile,
+		db:   db,
+	})
 
-	// If we have replication configured, start the replication listener.
-	if len(allowedReplicationPeers) > 0 {
-		rl := newReplicationListener(replicationListen, replCert, allowedReplicationPeers, db)
-		main.Add(rl)
-	}
-
-	// Start the main API server.
-	qs := newAPISrv(listen, cert, db, repl, useHTTP)
-	main.Add(qs)
-
-	// If we have a metrics port configured, start a metrics handler.
-	if metricsListen != "" {
-		go func() {
-			mux := http.NewServeMux()
-			mux.Handle("/metrics", promhttp.Handler())
-			log.Fatal(http.ListenAndServe(metricsListen, mux))
-		}()
-	}
-
-	// Engage!
-	main.Serve(context.Background())
+	globalStats.Reset()
+	main.Serve()
+}
+
+func getEnvDefault(key, def string) string {
+	if val := os.Getenv(key); val != "" {
+		return val
+	}
+	return def
+}
+
+func next(intv time.Duration) time.Duration {
+	t0 := time.Now()
+	t1 := t0.Add(intv).Truncate(intv)
+	return t1.Sub(t0)
 }

cmd/stdiscosrv/psql.go

@@ -0,0 +1,98 @@
+// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"database/sql"
+	"fmt"
+
+	_ "github.com/lib/pq"
+)
+
+func init() {
+	register("postgres", postgresSetup, postgresCompile)
+}
+
+func postgresSetup(db *sql.DB) error {
+	var err error
+
+	db.SetMaxIdleConns(4)
+	db.SetMaxOpenConns(8)
+
+	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Devices (
+		DeviceID CHAR(63) NOT NULL PRIMARY KEY,
+		Seen TIMESTAMP NOT NULL
+	)`)
+	if err != nil {
+		return err
+	}
+
+	var tmp string
+	row := db.QueryRow(`SELECT 'DevicesDeviceIDIndex'::regclass`)
+	if err = row.Scan(&tmp); err != nil {
+		_, err = db.Exec(`CREATE INDEX DevicesDeviceIDIndex ON Devices (DeviceID)`)
+	}
+	if err != nil {
+		return err
+	}
+
+	row = db.QueryRow(`SELECT 'DevicesSeenIndex'::regclass`)
+	if err = row.Scan(&tmp); err != nil {
+		_, err = db.Exec(`CREATE INDEX DevicesSeenIndex ON Devices (Seen)`)
+	}
+	if err != nil {
+		return err
+	}
+
+	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Addresses (
+		DeviceID CHAR(63) NOT NULL,
+		Seen TIMESTAMP NOT NULL,
+		Address VARCHAR(2048) NOT NULL
+	)`)
+	if err != nil {
+		return err
+	}
+
+	row = db.QueryRow(`SELECT 'AddressesDeviceIDSeenIndex'::regclass`)
+	if err = row.Scan(&tmp); err != nil {
+		_, err = db.Exec(`CREATE INDEX AddressesDeviceIDSeenIndex ON Addresses (DeviceID, Seen)`)
+	}
+	if err != nil {
+		return err
+	}
+
+	row = db.QueryRow(`SELECT 'AddressesDeviceIDAddressIndex'::regclass`)
+	if err = row.Scan(&tmp); err != nil {
+		_, err = db.Exec(`CREATE INDEX AddressesDeviceIDAddressIndex ON Addresses (DeviceID, Address)`)
+	}
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func postgresCompile(db *sql.DB) (map[string]*sql.Stmt, error) {
+	stmts := map[string]string{
+		"cleanAddress":  "DELETE FROM Addresses WHERE Seen < now() - '2 hour'::INTERVAL",
+		"cleanDevice":   fmt.Sprintf("DELETE FROM Devices WHERE Seen < now() - '%d hour'::INTERVAL", maxDeviceAge/3600),
+		"countAddress":  "SELECT count(*) FROM Addresses",
+		"countDevice":   "SELECT count(*) FROM Devices",
+		"insertAddress": "INSERT INTO Addresses (DeviceID, Seen, Address) VALUES ($1, now(), $2)",
+		"insertDevice":  "INSERT INTO Devices (DeviceID, Seen) VALUES ($1, now())",
+		"selectAddress": "SELECT Address FROM Addresses WHERE DeviceID=$1 AND Seen > now() - '1 hour'::INTERVAL ORDER BY random() LIMIT 16",
+		"selectDevice":  "SELECT Seen FROM Devices WHERE DeviceID=$1",
+		"updateAddress": "UPDATE Addresses SET Seen=now() WHERE DeviceID=$1 AND Address=$2",
+		"updateDevice":  "UPDATE Devices SET Seen=now() WHERE DeviceID=$1",
+	}
+
+	res := make(map[string]*sql.Stmt, len(stmts))
+	for key, stmt := range stmts {
+		prep, err := db.Prepare(stmt)
+		if err != nil {
+			return nil, err
+		}
+		res[key] = prep
+	}
+	return res, nil
+}

cmd/stdiscosrv/ql.go

@@ -0,0 +1,81 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"log"
+
+	"github.com/cznic/ql"
+)
+
+func init() {
+	ql.RegisterDriver()
+	register("ql", qlSetup, qlCompile)
+}
+
+func qlSetup(db *sql.DB) (err error) {
+	tx, err := db.Begin()
+	if err != nil {
+		return
+	}
+
+	defer func() {
+		if err == nil {
+			err = tx.Commit()
+		} else {
+			tx.Rollback()
+		}
+	}()
+
+	_, err = tx.Exec(`CREATE TABLE IF NOT EXISTS Devices (
+		DeviceID STRING NOT NULL,
+		Seen TIME NOT NULL
+	)`)
+	if err != nil {
+		return
+	}
+
+	if _, err = tx.Exec(`CREATE INDEX IF NOT EXISTS DevicesDeviceIDIndex ON Devices (DeviceID)`); err != nil {
+		return
+	}
+
+	_, err = tx.Exec(`CREATE TABLE IF NOT EXISTS Addresses (
+		DeviceID STRING NOT NULL,
+		Seen TIME NOT NULL,
+		Address STRING NOT NULL,
+	)`)
+	if err != nil {
+		return
+	}
+
+	_, err = tx.Exec(`CREATE INDEX IF NOT EXISTS AddressesDeviceIDAddressIndex ON Addresses (DeviceID, Address)`)
+	return
+}
+
+func qlCompile(db *sql.DB) (map[string]*sql.Stmt, error) {
+	stmts := map[string]string{
+		"cleanAddress":  `DELETE FROM Addresses WHERE Seen < now() - duration("2h")`,
+		"cleanDevice":   fmt.Sprintf(`DELETE FROM Devices WHERE Seen < now() - duration("%dh")`, maxDeviceAge/3600),
+		"countAddress":  "SELECT count(*) FROM Addresses",
+		"countDevice":   "SELECT count(*) FROM Devices",
+		"insertAddress": "INSERT INTO Addresses (DeviceID, Seen, Address) VALUES ($1, now(), $2)",
+		"insertDevice":  "INSERT INTO Devices (DeviceID, Seen) VALUES ($1, now())",
+		"selectAddress": `SELECT Address from Addresses WHERE DeviceID==$1 AND Seen > now() - duration("1h") LIMIT 16`,
+		"selectDevice":  "SELECT Seen FROM Devices WHERE DeviceID==$1",
+		"updateAddress": "UPDATE Addresses Seen=now() WHERE DeviceID==$1 AND Address==$2",
+		"updateDevice":  "UPDATE Devices Seen=now() WHERE DeviceID==$1",
+	}
+
+	res := make(map[string]*sql.Stmt, len(stmts))
+	for key, stmt := range stmts {
+		prep, err := db.Prepare(stmt)
+		if err != nil {
+			log.Println("Failed to compile", stmt)
+			return nil, err
+		}
+		res[key] = prep
+	}
+	return res, nil
+}

cmd/stdiscosrv/querysrv.go

@@ -0,0 +1,492 @@
+// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"database/sql"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"log"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/golang/groupcache/lru"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"golang.org/x/net/context"
+	"golang.org/x/time/rate"
+)
+
+type querysrv struct {
+	addr     string
+	db       *sql.DB
+	prep     map[string]*sql.Stmt
+	limiter  *safeCache
+	cert     tls.Certificate
+	listener net.Listener
+}
+
+type announcement struct {
+	Seen      time.Time `json:"seen"`
+	Addresses []string  `json:"addresses"`
+}
+
+type safeCache struct {
+	*lru.Cache
+	mut sync.Mutex
+}
+
+func (s *safeCache) Get(key string) (val interface{}, ok bool) {
+	s.mut.Lock()
+	val, ok = s.Cache.Get(key)
+	s.mut.Unlock()
+	return
+}
+
+func (s *safeCache) Add(key string, val interface{}) {
+	s.mut.Lock()
+	s.Cache.Add(key, val)
+	s.mut.Unlock()
+}
+
+type requestID int64
+
+func (i requestID) String() string {
+	return fmt.Sprintf("%016x", int64(i))
+}
+
+type contextKey int
+
+const idKey contextKey = iota
+
+func negCacheFor(lastSeen time.Time) int {
+	since := time.Since(lastSeen).Seconds()
+	if since >= maxDeviceAge {
+		return maxNegCache
+	}
+	if since < 0 {
+		// That's weird
+		return minNegCache
+	}
+
+	// Return a value linearly scaled from minNegCache (at zero seconds ago)
+	// to maxNegCache (at maxDeviceAge seconds ago).
+	r := since / maxDeviceAge
+	return int(minNegCache + r*(maxNegCache-minNegCache))
+}
+
+func (s *querysrv) Serve() {
+	s.limiter = &safeCache{
+		Cache: lru.New(lruSize),
+	}
+
+	if useHTTP {
+		listener, err := net.Listen("tcp", s.addr)
+		if err != nil {
+			log.Println("Listen:", err)
+			return
+		}
+		s.listener = listener
+	} else {
+		tlsCfg := &tls.Config{
+			Certificates:           []tls.Certificate{s.cert},
+			ClientAuth:             tls.RequestClientCert,
+			SessionTicketsDisabled: true,
+			MinVersion:             tls.VersionTLS12,
+			CipherSuites: []uint16{
+				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+			},
+		}
+
+		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
+		if err != nil {
+			log.Println("Listen:", err)
+			return
+		}
+		s.listener = tlsListener
+	}
+
+	http.HandleFunc("/v2/", s.handler)
+	http.HandleFunc("/ping", handlePing)
+
+	srv := &http.Server{
+		ReadTimeout:    5 * time.Second,
+		WriteTimeout:   5 * time.Second,
+		MaxHeaderBytes: 1 << 10,
+	}
+
+	if err := srv.Serve(s.listener); err != nil {
+		log.Println("Serve:", err)
+	}
+}
+
+var topCtx = context.Background()
+
+func (s *querysrv) handler(w http.ResponseWriter, req *http.Request) {
+	reqID := requestID(rand.Int63())
+	ctx := context.WithValue(topCtx, idKey, reqID)
+
+	if debug {
+		log.Println(reqID, req.Method, req.URL)
+	}
+
+	t0 := time.Now()
+	defer func() {
+		diff := time.Since(t0)
+		var comment string
+		if diff > time.Second {
+			comment = "(very slow request)"
+		} else if diff > 100*time.Millisecond {
+			comment = "(slow request)"
+		}
+		if comment != "" || debug {
+			log.Println(reqID, req.Method, req.URL, "completed in", diff, comment)
+		}
+	}()
+
+	var remoteIP net.IP
+	if useHTTP {
+		remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
+	} else {
+		addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
+		if err != nil {
+			log.Println("remoteAddr:", err)
+			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+			return
+		}
+		remoteIP = addr.IP
+	}
+
+	if s.limit(remoteIP) {
+		if debug {
+			log.Println(remoteIP, "is limited")
+		}
+		w.Header().Set("Retry-After", "60")
+		http.Error(w, "Too Many Requests", 429)
+		return
+	}
+
+	switch req.Method {
+	case "GET":
+		s.handleGET(ctx, w, req)
+	case "POST":
+		s.handlePOST(ctx, remoteIP, w, req)
+	default:
+		globalStats.Error()
+		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *querysrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
+	reqID := ctx.Value(idKey).(requestID)
+
+	deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
+	if err != nil {
+		if debug {
+			log.Println(reqID, "bad device param")
+		}
+		globalStats.Error()
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	var ann announcement
+
+	ann.Seen, err = s.getDeviceSeen(deviceID)
+	negCache := strconv.Itoa(negCacheFor(ann.Seen))
+	w.Header().Set("Retry-After", negCache)
+	w.Header().Set("Cache-Control", "public, max-age="+negCache)
+
+	if err != nil {
+		// The device is not in the database.
+		globalStats.Query()
+		http.Error(w, "Not Found", http.StatusNotFound)
+		return
+	}
+
+	t0 := time.Now()
+	ann.Addresses, err = s.getAddresses(ctx, deviceID)
+	if err != nil {
+		log.Println(reqID, "getAddresses:", err)
+		globalStats.Error()
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+	if debug {
+		log.Println(reqID, "getAddresses in", time.Since(t0))
+	}
+
+	globalStats.Query()
+
+	if len(ann.Addresses) == 0 {
+		http.Error(w, "Not Found", http.StatusNotFound)
+		return
+	}
+
+	globalStats.Answer()
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(ann)
+}
+
+func (s *querysrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
+	reqID := ctx.Value(idKey).(requestID)
+
+	rawCert := certificateBytes(req)
+	if rawCert == nil {
+		if debug {
+			log.Println(reqID, "no certificates")
+		}
+		globalStats.Error()
+		http.Error(w, "Forbidden", http.StatusForbidden)
+		return
+	}
+
+	var ann announcement
+	if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
+		if debug {
+			log.Println(reqID, "decode:", err)
+		}
+		globalStats.Error()
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	deviceID := protocol.NewDeviceID(rawCert)
+
+	// handleAnnounce returns *two* errors. The first indicates a problem with
+	// something the client posted to us. We should return a 400 Bad Request
+	// and not worry about it. The second indicates that the request was fine,
+	// but something internal messed up. We should log it and respond with a
+	// more apologetic 500 Internal Server Error.
+	userErr, internalErr := s.handleAnnounce(ctx, remoteIP, deviceID, ann.Addresses)
+	if userErr != nil {
+		if debug {
+			log.Println(reqID, "handleAnnounce:", userErr)
+		}
+		globalStats.Error()
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+	if internalErr != nil {
+		log.Println(reqID, "handleAnnounce:", internalErr)
+		globalStats.Error()
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+
+	globalStats.Announce()
+
+	// TODO: Slowly increase this for stable clients
+	w.Header().Set("Reannounce-After", "1800")
+
+	// We could return the lookup result here, but it's kind of unnecessarily
+	// expensive to go query the database again so we let the client decide to
+	// do a lookup if they really care.
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func (s *querysrv) Stop() {
+	s.listener.Close()
+}
+
+func (s *querysrv) handleAnnounce(ctx context.Context, remote net.IP, deviceID protocol.DeviceID, addresses []string) (userErr, internalErr error) {
+	reqID := ctx.Value(idKey).(requestID)
+
+	tx, err := s.db.Begin()
+	if err != nil {
+		internalErr = err
+		return
+	}
+
+	defer func() {
+		// Since we return from a bunch of different places, we handle
+		// rollback in the defer.
+		if internalErr != nil || userErr != nil {
+			tx.Rollback()
+		}
+	}()
+
+	for _, annAddr := range addresses {
+		uri, err := url.Parse(annAddr)
+		if err != nil {
+			userErr = err
+			return
+		}
+
+		host, port, err := net.SplitHostPort(uri.Host)
+		if err != nil {
+			userErr = err
+			return
+		}
+
+		ip := net.ParseIP(host)
+		if host == "" || ip.IsUnspecified() {
+			// Do not use IPv6 remote address if requested scheme is tcp4
+			if uri.Scheme == "tcp4" && remote.To4() == nil {
+				continue
+			}
+
+			// Do not use IPv4 remote address if requested scheme is tcp6
+			if uri.Scheme == "tcp6" && remote.To4() != nil {
+				continue
+			}
+
+			host = remote.String()
+		}
+
+		uri.Host = net.JoinHostPort(host, port)
+
+		if err := s.updateAddress(ctx, tx, deviceID, uri.String()); err != nil {
+			internalErr = err
+			return
+		}
+	}
+
+	if err := s.updateDevice(ctx, tx, deviceID); err != nil {
+		internalErr = err
+		return
+	}
+
+	t0 := time.Now()
+	internalErr = tx.Commit()
+	if debug {
+		log.Println(reqID, "commit in", time.Since(t0))
+	}
+	return
+}
+
+func (s *querysrv) limit(remote net.IP) bool {
+	key := remote.String()
+
+	bkt, ok := s.limiter.Get(key)
+	if ok {
+		bkt := bkt.(*rate.Limiter)
+		if !bkt.Allow() {
+			// Rate limit exceeded; ignore packet
+			return true
+		}
+	} else {
+		// limitAvg is in packets per ten seconds.
+		s.limiter.Add(key, rate.NewLimiter(rate.Limit(limitAvg)/10, limitBurst))
+	}
+
+	return false
+}
+
+func (s *querysrv) updateDevice(ctx context.Context, tx *sql.Tx, device protocol.DeviceID) error {
+	reqID := ctx.Value(idKey).(requestID)
+	t0 := time.Now()
+	res, err := tx.Stmt(s.prep["updateDevice"]).Exec(device.String())
+	if err != nil {
+		return err
+	}
+	if debug {
+		log.Println(reqID, "updateDevice in", time.Since(t0))
+	}
+
+	if rows, _ := res.RowsAffected(); rows == 0 {
+		t0 = time.Now()
+		_, err := tx.Stmt(s.prep["insertDevice"]).Exec(device.String())
+		if err != nil {
+			return err
+		}
+		if debug {
+			log.Println(reqID, "insertDevice in", time.Since(t0))
+		}
+	}
+
+	return nil
+}
+
+func (s *querysrv) updateAddress(ctx context.Context, tx *sql.Tx, device protocol.DeviceID, uri string) error {
+	res, err := tx.Stmt(s.prep["updateAddress"]).Exec(device.String(), uri)
+	if err != nil {
+		return err
+	}
+
+	if rows, _ := res.RowsAffected(); rows == 0 {
+		_, err := tx.Stmt(s.prep["insertAddress"]).Exec(device.String(), uri)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *querysrv) getAddresses(ctx context.Context, device protocol.DeviceID) ([]string, error) {
+	rows, err := s.prep["selectAddress"].Query(device.String())
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var res []string
+	for rows.Next() {
+		var addr string
+
+		err := rows.Scan(&addr)
+		if err != nil {
+			log.Println("Scan:", err)
+			continue
+		}
+		res = append(res, addr)
+	}
+
+	return res, nil
+}
+
+func (s *querysrv) getDeviceSeen(device protocol.DeviceID) (time.Time, error) {
+	row := s.prep["selectDevice"].QueryRow(device.String())
+	var seen time.Time
+	if err := row.Scan(&seen); err != nil {
+		return time.Time{}, err
+	}
+	return seen.In(time.UTC), nil
+}
+
+func handlePing(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(204)
+}
+
+func certificateBytes(req *http.Request) []byte {
+	if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
+		return req.TLS.PeerCertificates[0].Raw
+	}
+
+	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
+		bs := []byte(hdr)
+		// The certificate is in PEM format but with spaces for newlines. We
+		// need to reinstate the newlines for the PEM decoder. But we need to
+		// leave the spaces in the BEGIN and END lines - the first and last
+		// space - alone.
+		firstSpace := bytes.Index(bs, []byte(" "))
+		lastSpace := bytes.LastIndex(bs, []byte(" "))
+		for i := firstSpace + 1; i < lastSpace; i++ {
+			if bs[i] == ' ' {
+				bs[i] = '\n'
+			}
+		}
+		block, _ := pem.Decode(bs)
+		if block == nil {
+			// Decoding failed
+			return nil
+		}
+		return block.Bytes
+	}
+
+	return nil
+}

cmd/stdiscosrv/replication.go

@@ -1,315 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"context"
-	"crypto/tls"
-	"encoding/binary"
-	"fmt"
-	io "io"
-	"log"
-	"net"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-const replicationReadTimeout = time.Minute
-const replicationHeartbeatInterval = time.Second * 30
-
-type replicator interface {
-	send(key string, addrs []DatabaseAddress, seen int64)
-}
-
-// a replicationSender tries to connect to the remote address and provide
-// them with a feed of replication updates.
-type replicationSender struct {
-	dst        string
-	cert       tls.Certificate // our certificate
-	allowedIDs []protocol.DeviceID
-	outbox     chan ReplicationRecord
-}
-
-func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
-	return &replicationSender{
-		dst:        dst,
-		cert:       cert,
-		allowedIDs: allowedIDs,
-		outbox:     make(chan ReplicationRecord, replicationOutboxSize),
-	}
-}
-
-func (s *replicationSender) Serve(ctx context.Context) error {
-	// Sleep a little at startup. Peers often restart at the same time, and
-	// this avoid the service failing and entering backoff state
-	// unnecessarily, while also reducing the reconnect rate to something
-	// reasonable by default.
-	time.Sleep(2 * time.Second)
-
-	tlsCfg := &tls.Config{
-		Certificates:       []tls.Certificate{s.cert},
-		MinVersion:         tls.VersionTLS12,
-		InsecureSkipVerify: true,
-	}
-
-	// Dial the TLS connection.
-	conn, err := tls.Dial("tcp", s.dst, tlsCfg)
-	if err != nil {
-		log.Println("Replication connect:", err)
-		return err
-	}
-	defer func() {
-		conn.SetWriteDeadline(time.Now().Add(time.Second))
-		conn.Close()
-	}()
-
-	// Get the other side device ID.
-	remoteID, err := deviceID(conn)
-	if err != nil {
-		log.Println("Replication connect:", err)
-		return err
-	}
-
-	// Verify it's in the set of allowed device IDs.
-	if !deviceIDIn(remoteID, s.allowedIDs) {
-		log.Println("Replication connect: unexpected device ID:", remoteID)
-		return err
-	}
-
-	heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
-	defer heartBeatTicker.Stop()
-
-	// Send records.
-	buf := make([]byte, 1024)
-	for {
-		select {
-		case <-heartBeatTicker.C:
-			if len(s.outbox) > 0 {
-				// No need to send heartbeats if there are events/prevrious
-				// heartbeats to send, they will keep the connection alive.
-				continue
-			}
-			// Empty replication message is the heartbeat:
-			s.outbox <- ReplicationRecord{}
-
-		case rec := <-s.outbox:
-			// Buffer must hold record plus four bytes for size
-			size := rec.Size()
-			if len(buf) < size+4 {
-				buf = make([]byte, size+4)
-			}
-
-			// Record comes after the four bytes size
-			n, err := rec.MarshalTo(buf[4:])
-			if err != nil {
-				// odd to get an error here, but we haven't sent anything
-				// yet so it's not fatal
-				replicationSendsTotal.WithLabelValues("error").Inc()
-				log.Println("Replication marshal:", err)
-				continue
-			}
-			binary.BigEndian.PutUint32(buf, uint32(n))
-
-			// Send
-			conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
-			if _, err := conn.Write(buf[:4+n]); err != nil {
-				replicationSendsTotal.WithLabelValues("error").Inc()
-				log.Println("Replication write:", err)
-				// Yes, we are losing the replication event here.
-				return err
-			}
-			replicationSendsTotal.WithLabelValues("success").Inc()
-
-		case <-ctx.Done():
-			return nil
-		}
-	}
-}
-
-func (s *replicationSender) String() string {
-	return fmt.Sprintf("replicationSender(%q)", s.dst)
-}
-
-func (s *replicationSender) send(key string, ps []DatabaseAddress, _ int64) {
-	item := ReplicationRecord{
-		Key:       key,
-		Addresses: ps,
-	}
-
-	// The send should never block. The inbox is suitably buffered for at
-	// least a few seconds of stalls, which shouldn't happen in practice.
-	select {
-	case s.outbox <- item:
-	default:
-		replicationSendsTotal.WithLabelValues("drop").Inc()
-	}
-}
-
-// a replicationMultiplexer sends to multiple replicators
-type replicationMultiplexer []replicator
-
-func (m replicationMultiplexer) send(key string, ps []DatabaseAddress, seen int64) {
-	for _, s := range m {
-		// each send is nonblocking
-		s.send(key, ps, seen)
-	}
-}
-
-// replicationListener accepts incoming connections and reads replication
-// items from them. Incoming items are applied to the KV store.
-type replicationListener struct {
-	addr       string
-	cert       tls.Certificate
-	allowedIDs []protocol.DeviceID
-	db         database
-}
-
-func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
-	return &replicationListener{
-		addr:       addr,
-		cert:       cert,
-		allowedIDs: allowedIDs,
-		db:         db,
-	}
-}
-
-func (l *replicationListener) Serve(ctx context.Context) error {
-	tlsCfg := &tls.Config{
-		Certificates:       []tls.Certificate{l.cert},
-		ClientAuth:         tls.RequestClientCert,
-		MinVersion:         tls.VersionTLS12,
-		InsecureSkipVerify: true,
-	}
-
-	lst, err := tls.Listen("tcp", l.addr, tlsCfg)
-	if err != nil {
-		log.Println("Replication listen:", err)
-		return err
-	}
-	defer lst.Close()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return nil
-		default:
-		}
-
-		// Accept a connection
-		conn, err := lst.Accept()
-		if err != nil {
-			log.Println("Replication accept:", err)
-			return err
-		}
-
-		// Figure out the other side device ID
-		remoteID, err := deviceID(conn.(*tls.Conn))
-		if err != nil {
-			log.Println("Replication accept:", err)
-			conn.SetWriteDeadline(time.Now().Add(time.Second))
-			conn.Close()
-			continue
-		}
-
-		// Verify it is in the set of allowed device IDs
-		if !deviceIDIn(remoteID, l.allowedIDs) {
-			log.Println("Replication accept: unexpected device ID:", remoteID)
-			conn.SetWriteDeadline(time.Now().Add(time.Second))
-			conn.Close()
-			continue
-		}
-
-		go l.handle(ctx, conn)
-	}
-}
-
-func (l *replicationListener) String() string {
-	return fmt.Sprintf("replicationListener(%q)", l.addr)
-}
-
-func (l *replicationListener) handle(ctx context.Context, conn net.Conn) {
-	defer func() {
-		conn.SetWriteDeadline(time.Now().Add(time.Second))
-		conn.Close()
-	}()
-
-	buf := make([]byte, 1024)
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		default:
-		}
-
-		conn.SetReadDeadline(time.Now().Add(replicationReadTimeout))
-
-		// First four bytes are the size
-		if _, err := io.ReadFull(conn, buf[:4]); err != nil {
-			log.Println("Replication read size:", err)
-			replicationRecvsTotal.WithLabelValues("error").Inc()
-			return
-		}
-
-		// Read the rest of the record
-		size := int(binary.BigEndian.Uint32(buf[:4]))
-		if len(buf) < size {
-			buf = make([]byte, size)
-		}
-
-		if size == 0 {
-			// Heartbeat, ignore
-			continue
-		}
-
-		if _, err := io.ReadFull(conn, buf[:size]); err != nil {
-			log.Println("Replication read record:", err)
-			replicationRecvsTotal.WithLabelValues("error").Inc()
-			return
-		}
-
-		// Unmarshal
-		var rec ReplicationRecord
-		if err := rec.Unmarshal(buf[:size]); err != nil {
-			log.Println("Replication unmarshal:", err)
-			replicationRecvsTotal.WithLabelValues("error").Inc()
-			continue
-		}
-
-		// Store
-		l.db.merge(rec.Key, rec.Addresses, rec.Seen)
-		replicationRecvsTotal.WithLabelValues("success").Inc()
-	}
-}
-
-func deviceID(conn *tls.Conn) (protocol.DeviceID, error) {
-	// Handshake may not be complete on the server side yet, which we need
-	// to get the client certificate.
-	if !conn.ConnectionState().HandshakeComplete {
-		if err := conn.Handshake(); err != nil {
-			return protocol.DeviceID{}, err
-		}
-	}
-
-	// We expect exactly one certificate.
-	certs := conn.ConnectionState().PeerCertificates
-	if len(certs) != 1 {
-		return protocol.DeviceID{}, fmt.Errorf("unexpected number of certificates (%d != 1)", len(certs))
-	}
-
-	return protocol.NewDeviceID(certs[0].Raw), nil
-}
-
-func deviceIDIn(id protocol.DeviceID, ids []protocol.DeviceID) bool {
-	for _, candidate := range ids {
-		if id == candidate {
-			return true
-		}
-	}
-	return false
-}

cmd/stdiscosrv/scripts/preinst

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-addgroup --system syncthing
-adduser --system --home /var/lib/syncthing-discosrv --ingroup syncthing syncthing-discosrv

cmd/stdiscosrv/stats.go

@@ -1,124 +1,141 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
+// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
 
 package main
 
 import (
+	"bytes"
+	"database/sql"
+	"fmt"
+	"io/ioutil"
+	"log"
 	"os"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/collectors"
+	"sync/atomic"
+	"time"
 )
 
-var (
-	apiRequestsTotal = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "api_requests_total",
-			Help:      "Number of API requests.",
-		}, []string{"type", "result"})
-	apiRequestsSeconds = prometheus.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Namespace:  "syncthing",
-			Subsystem:  "discovery",
-			Name:       "api_requests_seconds",
-			Help:       "Latency of API requests.",
-			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-		}, []string{"type"})
+type stats struct {
+	// Incremented atomically
+	announces int64
+	queries   int64
+	answers   int64
+	errors    int64
+}
 
-	lookupRequestsTotal = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "lookup_requests_total",
-			Help:      "Number of lookup requests.",
-		}, []string{"result"})
-	announceRequestsTotal = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "announcement_requests_total",
-			Help:      "Number of announcement requests.",
-		}, []string{"result"})
+func (s *stats) Announce() {
+	atomic.AddInt64(&s.announces, 1)
+}
 
-	replicationSendsTotal = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "replication_sends_total",
-			Help:      "Number of replication sends.",
-		}, []string{"result"})
-	replicationRecvsTotal = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "replication_recvs_total",
-			Help:      "Number of replication receives.",
-		}, []string{"result"})
+func (s *stats) Query() {
+	atomic.AddInt64(&s.queries, 1)
+}
 
-	databaseKeys = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "database_keys",
-			Help:      "Number of database keys at last count.",
-		}, []string{"category"})
-	databaseStatisticsSeconds = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "database_statistics_seconds",
-			Help:      "Time spent running the statistics routine.",
-		})
+func (s *stats) Answer() {
+	atomic.AddInt64(&s.answers, 1)
+}
 
-	databaseOperations = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: "syncthing",
-			Subsystem: "discovery",
-			Name:      "database_operations_total",
-			Help:      "Number of database operations.",
-		}, []string{"operation", "result"})
-	databaseOperationSeconds = prometheus.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Namespace:  "syncthing",
-			Subsystem:  "discovery",
-			Name:       "database_operation_seconds",
-			Help:       "Latency of database operations.",
-			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-		}, []string{"operation"})
-)
+func (s *stats) Error() {
+	atomic.AddInt64(&s.errors, 1)
+}
 
-const (
-	dbOpGet             = "get"
-	dbOpPut             = "put"
-	dbOpMerge           = "merge"
-	dbOpDelete          = "delete"
-	dbResSuccess        = "success"
-	dbResNotFound       = "not_found"
-	dbResError          = "error"
-	dbResUnmarshalError = "unmarsh_err"
-)
-
-func init() {
-	prometheus.MustRegister(apiRequestsTotal, apiRequestsSeconds,
-		lookupRequestsTotal, announceRequestsTotal,
-		replicationSendsTotal, replicationRecvsTotal,
-		databaseKeys, databaseStatisticsSeconds,
-		databaseOperations, databaseOperationSeconds)
-
-	processCollectorOpts := collectors.ProcessCollectorOpts{
-		Namespace: "syncthing_discovery",
-		PidFn: func() (int, error) {
-			return os.Getpid(), nil
-		},
+// Reset returns a copy of the current stats and resets the counters to
+// zero.
+func (s *stats) Reset() stats {
+	// Create a copy of the stats using atomic reads
+	copy := stats{
+		announces: atomic.LoadInt64(&s.announces),
+		queries:   atomic.LoadInt64(&s.queries),
+		answers:   atomic.LoadInt64(&s.answers),
+		errors:    atomic.LoadInt64(&s.errors),
 	}
 
-	prometheus.MustRegister(
-		collectors.NewProcessCollector(processCollectorOpts),
-	)
+	// Reset the stats by subtracting the values that we copied
+	atomic.AddInt64(&s.announces, -copy.announces)
+	atomic.AddInt64(&s.queries, -copy.queries)
+	atomic.AddInt64(&s.answers, -copy.answers)
+	atomic.AddInt64(&s.errors, -copy.errors)
 
+	return copy
+}
+
+type statssrv struct {
+	intv time.Duration
+	file string
+	db   *sql.DB
+}
+
+func (s *statssrv) Serve() {
+	lastReset := time.Now()
+	for {
+		time.Sleep(next(s.intv))
+
+		stats := globalStats.Reset()
+		d := time.Since(lastReset).Seconds()
+		lastReset = time.Now()
+
+		log.Printf("Stats: %.02f announces/s, %.02f queries/s, %.02f answers/s, %.02f errors/s",
+			float64(stats.announces)/d, float64(stats.queries)/d, float64(stats.answers)/d, float64(stats.errors)/d)
+
+		if s.file != "" {
+			s.writeToFile(stats, d)
+		}
+	}
+}
+
+func (s *statssrv) Stop() {
+	panic("stop unimplemented")
+}
+
+func (s *statssrv) writeToFile(stats stats, secs float64) {
+	newLine := []byte("\n")
+
+	var addrs int
+	row := s.db.QueryRow("SELECT COUNT(*) FROM Addresses")
+	if err := row.Scan(&addrs); err != nil {
+		log.Println("stats query:", err)
+		return
+	}
+
+	fd, err := os.OpenFile(s.file, os.O_RDWR|os.O_CREATE, 0666)
+	if err != nil {
+		log.Println("stats file:", err)
+		return
+	}
+	defer func() {
+		err = fd.Close()
+		if err != nil {
+			log.Println("stats file:", err)
+		}
+	}()
+
+	bs, err := ioutil.ReadAll(fd)
+	if err != nil {
+		log.Println("stats file:", err)
+		return
+	}
+	lines := bytes.Split(bytes.TrimSpace(bs), newLine)
+	if len(lines) > 12 {
+		lines = lines[len(lines)-12:]
+	}
+
+	latest := fmt.Sprintf("%v: %6d addresses, %8.02f announces/s, %8.02f queries/s, %8.02f answers/s, %8.02f errors/s\n",
+		time.Now().UTC().Format(time.RFC3339), addrs,
+		float64(stats.announces)/secs, float64(stats.queries)/secs, float64(stats.answers)/secs, float64(stats.errors)/secs)
+	lines = append(lines, []byte(latest))
+
+	_, err = fd.Seek(0, 0)
+	if err != nil {
+		log.Println("stats file:", err)
+		return
+	}
+	err = fd.Truncate(0)
+	if err != nil {
+		log.Println("stats file:", err)
+		return
+	}
+
+	_, err = fd.Write(bytes.Join(lines, newLine))
+	if err != nil {
+		log.Println("stats file:", err)
+		return
+	}
 }

cmd/stevents/main.go

@@ -28,21 +28,16 @@ func main() {
 	log.SetFlags(0)
 
 	target := flag.String("target", "localhost:8384", "Target Syncthing instance")
-	types := flag.String("types", "", "Filter for specific event types (comma-separated)")
 	apikey := flag.String("apikey", "", "Syncthing API key")
 	flag.Parse()
 
 	if *apikey == "" {
 		log.Fatal("Must give -apikey argument")
 	}
-	var eventsArg string
-	if len(*types) > 0 {
-		eventsArg = "&events=" + *types
-	}
 
 	since := 0
 	for {
-		req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d%s", *target, since, eventsArg), nil)
+		req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d", *target, since), nil)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/stfileinfo/main.go

@@ -68,8 +68,8 @@ func main() {
 		}
 
 		blockSize := int(fi.Size())
-		if *standardBlocks || blockSize < protocol.MinBlockSize {
-			blockSize = protocol.BlockSize(fi.Size())
+		if *standardBlocks || blockSize < protocol.BlockSize {
+			blockSize = protocol.BlockSize
 		}
 		bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
 		if err != nil {

cmd/stfinddevice/main.go

@@ -7,7 +7,6 @@
 package main
 
 import (
-	"context"
 	"crypto/tls"
 	"errors"
 	"flag"
@@ -18,7 +17,6 @@ import (
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
-	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -84,7 +82,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 }
 
 func checkServer(deviceID protocol.DeviceID, server string) checkResult {
-	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger, nil)
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
 	if err != nil {
 		return checkResult{error: err}
 	}
@@ -96,7 +94,7 @@ func checkServer(deviceID protocol.DeviceID, server string) checkResult {
 	})
 
 	go func() {
-		addresses, err := disco.Lookup(context.Background(), deviceID)
+		addresses, err := disco.Lookup(deviceID)
 		res <- checkResult{addresses: addresses, error: err}
 	}()
 

cmd/stfindignored/main.go

@@ -1,44 +0,0 @@
-// Copyright (C) 2018 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// Command stfindignored lists ignored files under a given folder root.
-package main
-
-import (
-	"flag"
-	"fmt"
-	"os"
-
-	"github.com/syncthing/syncthing/lib/fs"
-	"github.com/syncthing/syncthing/lib/ignore"
-)
-
-func main() {
-	flag.Parse()
-	root := flag.Arg(0)
-	if root == "" {
-		root = "."
-	}
-
-	vfs := fs.NewWalkFilesystem(fs.NewFilesystem(fs.FilesystemTypeBasic, root))
-
-	ign := ignore.New(vfs)
-	if err := ign.Load(".stignore"); err != nil {
-		fmt.Fprintf(os.Stderr, "Fatal: loading ignores: %v\n", err)
-		os.Exit(1)
-	}
-
-	vfs.Walk(".", func(path string, info fs.FileInfo, err error) error {
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "Warning: %s: %v\n", path, err)
-			return fs.SkipDir
-		}
-		if ign.Match(path).IsIgnored() {
-			fmt.Println(path)
-		}
-		return nil
-	})
-}

cmd/stgenfiles/main.go

@@ -66,7 +66,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
 }
 
 func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
-	src := io.LimitReader(&infiniteReader{fd}, s)
+	src := io.LimitReader(&inifiteReader{fd}, s)
 	dst, err := os.Create(p1)
 	if err != nil {
 		return err
@@ -82,7 +82,7 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
 		return err
 	}
 
-	os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
+	_ = os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
 
 	t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
 	return os.Chtimes(p1, t, t)
@@ -105,11 +105,11 @@ func readRand(bs []byte) (int, error) {
 	return len(bs), nil
 }
 
-type infiniteReader struct {
+type inifiteReader struct {
 	rd io.ReadSeeker
 }
 
-func (i *infiniteReader) Read(bs []byte) (int, error) {
+func (i *inifiteReader) Read(bs []byte) (int, error) {
 	n, err := i.rd.Read(bs)
 	if err == io.EOF {
 		err = nil

cmd/stindex/dump.go

@@ -0,0 +1,83 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/binary"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func dump(ldb *db.Instance) {
+	it := ldb.NewIterator(nil, nil)
+	for it.Next() {
+		key := it.Key()
+		switch key[0] {
+		case db.KeyTypeDevice:
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+			fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
+
+			var f protocol.FileInfo
+			err := f.Unmarshal(it.Value())
+			if err != nil {
+				log.Fatal(err)
+			}
+			fmt.Printf(" V:%v\n", f)
+
+		case db.KeyTypeGlobal:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			var flv db.VersionList
+			flv.Unmarshal(it.Value())
+			fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
+
+		case db.KeyTypeBlock:
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[1+4 : 1+4+32]
+			name := nulString(key[1+4+32:])
+			fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
+
+		case db.KeyTypeDeviceStatistic:
+			fmt.Printf("[dstat] K:%x V:%x\n", it.Key(), it.Value())
+
+		case db.KeyTypeFolderStatistic:
+			fmt.Printf("[fstat] K:%x V:%x\n", it.Key(), it.Value())
+
+		case db.KeyTypeVirtualMtime:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			val := it.Value()
+			var real, virt time.Time
+			real.UnmarshalBinary(val[:len(val)/2])
+			virt.UnmarshalBinary(val[len(val)/2:])
+			fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, real, virt)
+
+		case db.KeyTypeFolderIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
+
+		case db.KeyTypeDeviceIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			val := it.Value()
+			if len(val) == 0 {
+				fmt.Printf("[deviceidx] K:%d V:<nil>\n", key)
+			} else {
+				dev := protocol.DeviceIDFromBytes(val)
+				fmt.Printf("[deviceidx] K:%d V:%s\n", key, dev)
+			}
+
+		default:
+			fmt.Printf("[???]\n  %x\n  %x\n", it.Key(), it.Value())
+		}
+	}
+}

cmd/stindex/dumpsize.go

@@ -0,0 +1,94 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"container/heap"
+	"encoding/binary"
+	"fmt"
+
+	"github.com/syncthing/syncthing/lib/db"
+)
+
+type SizedElement struct {
+	key  string
+	size int
+}
+
+type ElementHeap []SizedElement
+
+func (h ElementHeap) Len() int           { return len(h) }
+func (h ElementHeap) Less(i, j int) bool { return h[i].size > h[j].size }
+func (h ElementHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+
+func (h *ElementHeap) Push(x interface{}) {
+	*h = append(*h, x.(SizedElement))
+}
+
+func (h *ElementHeap) Pop() interface{} {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	*h = old[0 : n-1]
+	return x
+}
+
+func dumpsize(ldb *db.Instance) {
+	h := &ElementHeap{}
+	heap.Init(h)
+
+	it := ldb.NewIterator(nil, nil)
+	var ele SizedElement
+	for it.Next() {
+		key := it.Key()
+		switch key[0] {
+		case db.KeyTypeDevice:
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+			ele.key = fmt.Sprintf("DEVICE:%d:%d:%s", folder, device, name)
+
+		case db.KeyTypeGlobal:
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			ele.key = fmt.Sprintf("GLOBAL:%d:%s", folder, name)
+
+		case db.KeyTypeBlock:
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[1+4 : 1+4+32]
+			name := nulString(key[1+4+32:])
+			ele.key = fmt.Sprintf("BLOCK:%d:%x:%s", folder, hash, name)
+
+		case db.KeyTypeDeviceStatistic:
+			ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])
+
+		case db.KeyTypeFolderStatistic:
+			ele.key = fmt.Sprintf("FOLDERSTATS:%s", key[1:])
+
+		case db.KeyTypeVirtualMtime:
+			ele.key = fmt.Sprintf("MTIME:%s", key[1:])
+
+		case db.KeyTypeFolderIdx:
+			id := binary.BigEndian.Uint32(key[1:])
+			ele.key = fmt.Sprintf("FOLDERIDX:%d", id)
+
+		case db.KeyTypeDeviceIdx:
+			id := binary.BigEndian.Uint32(key[1:])
+			ele.key = fmt.Sprintf("DEVICEIDX:%d", id)
+
+		default:
+			ele.key = fmt.Sprintf("UNKNOWN:%x", key)
+		}
+		ele.size = len(it.Value())
+		heap.Push(h, ele)
+	}
+
+	for h.Len() > 0 {
+		ele = heap.Pop(h).(SizedElement)
+		fmt.Println(ele.key, ele.size)
+	}
+}

cmd/stindex/main.go

@@ -0,0 +1,47 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/syncthing/syncthing/lib/db"
+)
+
+func main() {
+	var mode string
+	log.SetFlags(0)
+	log.SetOutput(os.Stdout)
+
+	flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize")
+
+	flag.Parse()
+
+	path := flag.Arg(0)
+	if path == "" {
+		path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
+	}
+
+	fmt.Println("Path:", path)
+
+	ldb, err := db.Open(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if mode == "dump" {
+		dump(ldb)
+	} else if mode == "dumpsize" {
+		dumpsize(ldb)
+	} else {
+		fmt.Println("Unknown mode")
+	}
+}

cmd/stindex/util.go

@@ -0,0 +1,52 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/syncthing/syncthing/lib/osutil"
+)
+
+func nulString(bs []byte) string {
+	for i := range bs {
+		if bs[i] == 0 {
+			return string(bs[:i])
+		}
+	}
+	return string(bs)
+}
+
+func defaultConfigDir() string {
+	switch runtime.GOOS {
+	case "windows":
+		if p := os.Getenv("LocalAppData"); p != "" {
+			return filepath.Join(p, "Syncthing")
+		}
+		return filepath.Join(os.Getenv("AppData"), "Syncthing")
+
+	case "darwin":
+		dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
+		if err != nil {
+			log.Fatal(err)
+		}
+		return dir
+
+	default:
+		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
+			return filepath.Join(xdgCfg, "syncthing")
+		}
+		dir, err := osutil.ExpandTilde("~/.config/syncthing")
+		if err != nil {
+			log.Fatal(err)
+		}
+		return dir
+	}
+}