Compare commits

..

1 Commits

Author SHA1 Message Date
Jakob Borg
1f09488a0f lib/model, lib/versioner: Prevent symlink attack via versioning (fixes #4286)
Prior to this, the following is possible:

- Create a symlink "foo -> /somewhere", it gets synced
- Delete "foo", it gets versioned
- Create "foo/bar", it gets synced
- Delete "foo/bar", it gets versioned in "/somewhere/bar"

With this change, versioners should never version symlinks.
2017-08-08 08:02:33 +02:00
2905 changed files with 1149354 additions and 220017 deletions

View File

@@ -1,20 +0,0 @@
comment: false
coverage:
range: "40...100"
precision: 1
status:
patch:
default:
informational: true
project:
default:
informational: true
github_checks:
annotations: false
ignore:
- "**.pb.go"
- "**_mocked.go"
- "**/mocks/*"

View File

@@ -1,12 +0,0 @@
version = 1
exclude_patterns = ["**/*.pb.go"]
test_patterns = ["**/*_test.go"]
[[analyzers]]
name = "go"
enabled = true
[analyzers.meta]
import_paths = ["github.com/syncthing/syncthing"]
build_tags = ["noassets"]

2
.github/CODEOWNERS vendored
View File

@@ -1,2 +0,0 @@
/AUTHORS @calmh
/*.md @calmh

11
.github/FUNDING.yml vendored
View File

@@ -1,11 +0,0 @@
github: syncthing
custom: "https://syncthing.net/donations/"
# patreon: # Replace with a single Patreon username
# open_collective: # Replace with a single Open Collective username
# ko_fi: # Replace with a single Ko-fi username
# tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
# community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
# liberapay: # Replace with a single Liberapay username
# issuehunt: # Replace with a single IssueHunt username
# otechie: # Replace with a single Otechie username

View File

@@ -1,28 +0,0 @@
name: Feature request
description: File a new feature request
labels: ["enhancement", "needs-triage"]
body:
- type: textarea
id: feature
attributes:
label: Feature description
description: Please describe the behavior you'd like to see.
validations:
required: true
- type: textarea
id: problem-usecase
attributes:
label: Problem or use case
description: Please explain which problem this would solve, or what the use case is for the feature. Keep in mind that it's more likely to be implemented if it's generally useful for a larger number of users.
validations:
required: true
- type: textarea
id: alternatives
attributes:
label: Alternatives or workarounds
description: Please describe any alternatives or workarounds you have considered and, possibly, rejected.
validations:
required: true

View File

@@ -1,51 +0,0 @@
name: Bug report
description: If you're actually looking for support instead, see "I need help / I have a question".
labels: ["bug", "needs-triage"]
body:
- type: markdown
attributes:
value: |
:no_entry_sign: If you want to report a security issue, please see [our Security Policy](https://syncthing.net/security/) and do not report the issue here.
:interrobang: If you are not sure if there is a bug, but something isn't working right and you need help, please [use the forum](https://forum.syncthing.net/).
- type: textarea
id: what-happened
attributes:
label: What happened?
description: Also tell us, what did you expect to happen, and any steps we might use to reproduce the problem.
placeholder: Tell us what you see!
validations:
required: true
- type: input
id: version
attributes:
label: Syncthing version
description: What version of Syncthing are you running?
placeholder: v1.27.4
validations:
required: true
- type: input
id: platform
attributes:
label: Platform & operating system
description: On what platform(s) are you seeing the problem?
placeholder: Linux arm64
validations:
required: true
- type: input
id: browser
attributes:
label: Browser version
description: If the problem is related to the GUI, describe your browser and version.
placeholder: Safari 17.3.1
- type: textarea
id: logs
attributes:
label: Relevant log output
description: Please copy and paste any relevant log output or crash backtrace. This will be automatically formatted into code, so no need for backticks.
render: shell

View File

@@ -1,8 +0,0 @@
blank_issues_enabled: false
contact_links:
- name: I need help / I have a question
url: https://forum.syncthing.net/
about: Ask questions, get support, and discuss with other community members.
- name: Android issues
url: https://github.com/syncthing/syncthing-android/issues/
about: The Android app has its own issue tracker.

View File

@@ -1,27 +0,0 @@
### Purpose
Describe the purpose of this change. If there is an existing issue that is
resolved by this pull request, ensure that the commit subject is on the form
`Some short description (fixes #1234)` where 1234 is the issue number.
### Testing
Describe what testing has been done, and how the reviewer can test the change
if new tests are not included.
### Screenshots
If this is a GUI change, include screenshots of the change. If not, please
feel free to just delete this section.
### Documentation
If this is a user visible change (including API and protocol changes), add a link here
to the corresponding pull request on https://github.com/syncthing/docs or describe
the documentation changes necessary.
## Authorship
Your name and email will be added automatically to the AUTHORS file
based on the commit metadata.

10
.github/SECURITY.md vendored
View File

@@ -1,10 +0,0 @@
## Reporting a Vulnerability
If you believe that you've found a Syncthing-related security vulnerability,
please report it by sending email to the address security@syncthing.net. The
[PGP key for security@syncthing.net
(B683AD7B76CAB013)](https://syncthing.net/security-key.txt) can be used to
send encrypted mail or to verify responses received from that address.
You can read more about Syncthing security at
https://syncthing.net/security/.

View File

@@ -1,13 +0,0 @@
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: weekly
open-pull-requests-limit: 10
- package-ecosystem: "gomod"
directory: "/"
schedule:
interval: weekly
open-pull-requests-limit: 10

View File

@@ -1,74 +0,0 @@
name: Build Infrastructure Images
on:
push:
branches:
- infrastructure
- infra-*
env:
GO_VERSION: "~1.22.3"
CGO_ENABLED: "0"
BUILD_USER: docker
BUILD_HOST: github.syncthing.net
jobs:
docker-syncthing:
name: Build and push Docker images
if: github.repository == 'syncthing/syncthing'
runs-on: ubuntu-latest
environment: docker
strategy:
matrix:
pkg:
- stcrashreceiver
- strelaypoolsrv
- stupgrades
- ursrv
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
check-latest: true
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build binaries
run: |
for arch in arm64 amd64; do
go run build.go -goos linux -goarch "$arch" build ${{ matrix.pkg }}
mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-"$arch"
done
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Set Docker tags (all branches)
run: |
tags=syncthing/${{ matrix.pkg }}:${{ github.sha }}
echo "TAGS=$tags" >> $GITHUB_ENV
- name: Set Docker tags (latest)
if: github.ref == 'refs/heads/infrastructure'
run: |
tags=syncthing/${{ matrix.pkg }}:latest,${{ env.TAGS }}
echo "TAGS=$tags" >> $GITHUB_ENV
- name: Build and push
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile.${{ matrix.pkg }}
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ env.TAGS }}
labels: |
org.opencontainers.image.revision=${{ github.sha }}

View File

@@ -1,848 +0,0 @@
name: Build Syncthing
on:
pull_request:
push:
schedule:
# Run nightly build at 05:00 UTC
- cron: '00 05 * * *'
workflow_dispatch:
env:
# The go version to use for builds. We set check-latest to true when
# installing, so we get the latest patch version that matches the
# expression.
GO_VERSION: "~1.22.3"
# Optimize compatibility on the slow archictures.
GO386: softfloat
GOARM: "5"
GOMIPS: softfloat
# Avoid hilarious amounts of obscuring log output when running tests.
LOGGER_DISCARD: "1"
# Our build metadata
BUILD_USER: builder
BUILD_HOST: github.syncthing.net
# A note on actions and third party code... The actions under actions/ (like
# `uses: actions/checkout`) are maintained by GitHub, and we need to trust
# GitHub to maintain their code and infrastructure or we're in deep shit in
# general. The same doesn't necessarily apply to other actions authors, so
# some care needs to be taken when adding steps, especially in the paths
# that lead up to code being packaged and signed.
jobs:
#
# Tests for all platforms. Runs a matrix build on Windows, Linux and Mac,
# with the list of expected supported Go versions (current, previous).
#
build-test:
name: Build and test
strategy:
fail-fast: false
matrix:
runner: ["windows-latest", "ubuntu-latest", "macos-latest"]
# The oldest version in this list should match what we have in our go.mod.
# Variables don't seem to be supported here, or we could have done something nice.
go: ["~1.21.7", "~1.22.3"]
runs-on: ${{ matrix.runner }}
steps:
- name: Set git to use LF
if: matrix.runner == 'windows-latest'
# Without this, the Windows checkout will happen with CRLF line
# endings, which is fine for the source code but messes up tests
# that depend on data on disk being as expected. Ideally, those
# tests should be fixed, but not today.
run: |
git config --global core.autocrlf false
git config --global core.eol lf
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go }}
cache: true
check-latest: true
- name: Build
run: |
go run build.go
- name: Install go-test-json-to-loki
run: |
go install calmh.dev/go-test-json-to-loki@latest
- name: Test
run: |
go version
go run build.go test | go-test-json-to-loki
env:
GOFLAGS: "-json"
LOKI_URL: ${{ vars.LOKI_URL }}
LOKI_USER: ${{ vars.LOKI_USER }}
LOKI_PASSWORD: ${{ secrets.LOKI_PASSWORD }}
LOKI_LABELS: "go=${{ matrix.go }},runner=${{ matrix.runner }},repo=${{ github.repository }},ref=${{ github.ref }}"
#
# Meta checks for formatting, copyright, etc
#
correctness:
name: Check correctness
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: false
check-latest: true
- name: Check correctness
run: |
go test -v ./meta
#
# The basic checks job is a virtual one that depends on the matrix tests,
# the correctness checks, and various builds that we always do. This makes
# it easy to have the PR process have a single test as a gatekeeper for
# merging, instead of having to add all the matrix tests and update them
# each time the version changes. (The top level test is not available for
# choosing there, only the matrix "children".)
#
basics:
name: Basic checks passed
runs-on: ubuntu-latest
needs:
- build-test
- correctness
- package-linux
- package-cross
- package-source
- package-debian
- govulncheck
steps:
- uses: actions/checkout@v4
#
# Windows
#
package-windows:
name: Package for Windows
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
environment: signing
runs-on: windows-latest
steps:
- name: Set git to use LF
# Without this, the checkout will happen with CRLF line endings,
# which is fine for the source code but messes up tests that depend
# on data on disk being as expected. Ideally, those tests should be
# fixed, but not today.
run: |
git config --global core.autocrlf false
git config --global core.eol lf
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: false
check-latest: true
- name: Get actual Go version
run: |
go version
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
- uses: actions/cache@v4
with:
path: |
~\AppData\Local\go-build
~\go\pkg\mod
key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
- name: Install dependencies
run: |
go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@v1.4.0
- name: Create packages
run: |
go run build.go -goarch amd64 zip
go run build.go -goarch arm zip
go run build.go -goarch arm64 zip
go run build.go -goarch 386 zip
env:
CGO_ENABLED: "0"
CODESIGN_SIGNTOOL: ${{ secrets.CODESIGN_SIGNTOOL }}
CODESIGN_CERTIFICATE_BASE64: ${{ secrets.CODESIGN_CERTIFICATE_BASE64 }}
CODESIGN_CERTIFICATE_PASSWORD: ${{ secrets.CODESIGN_CERTIFICATE_PASSWORD }}
CODESIGN_TIMESTAMP_SERVER: ${{ secrets.CODESIGN_TIMESTAMP_SERVER }}
- name: Archive artifacts
uses: actions/upload-artifact@v4
with:
name: packages-windows
path: syncthing-windows-*.zip
#
# Linux
#
package-linux:
name: Package for Linux
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: false
check-latest: true
- name: Get actual Go version
run: |
go version
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
- uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
- name: Create packages
run: |
archs=$(go tool dist list | grep linux | sed 's#linux/##')
for goarch in $archs ; do
go run build.go -goarch "$goarch" tar
done
env:
CGO_ENABLED: "0"
- name: Archive artifacts
uses: actions/upload-artifact@v4
with:
name: packages-linux
path: syncthing-linux-*.tar.gz
#
# macOS
#
package-macos:
name: Package for macOS
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
environment: signing
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: false
check-latest: true
- name: Get actual Go version
run: |
go version
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
- uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
- name: Import signing certificate
run: |
# Set up a run-specific keychain, making it available for the
# `codesign` tool.
umask 066
KEYCHAIN_PATH=$RUNNER_TEMP/codesign.keychain
KEYCHAIN_PASSWORD=$(uuidgen)
security create-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
security default-keychain -s "$KEYCHAIN_PATH"
security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
security set-keychain-settings -lut 21600 "$KEYCHAIN_PATH"
# Import the certificate
CERTIFICATE_PATH=$RUNNER_TEMP/codesign.p12
echo "$DEVELOPER_ID_CERTIFICATE_BASE64" | base64 -d -o "$CERTIFICATE_PATH"
security import "$CERTIFICATE_PATH" -k "$KEYCHAIN_PATH" -P "$DEVELOPER_ID_CERTIFICATE_PASSWORD" -T /usr/bin/codesign -T /usr/bin/productsign
security set-key-partition-list -S apple-tool:,apple: -s -k actions "$KEYCHAIN_PATH"
# Set the codesign identity for following steps
echo "CODESIGN_IDENTITY=$CODESIGN_IDENTITY" >> $GITHUB_ENV
env:
DEVELOPER_ID_CERTIFICATE_BASE64: ${{ secrets.DEVELOPER_ID_CERTIFICATE_BASE64 }}
DEVELOPER_ID_CERTIFICATE_PASSWORD: ${{ secrets.DEVELOPER_ID_CERTIFICATE_PASSWORD }}
CODESIGN_IDENTITY: ${{ secrets.CODESIGN_IDENTITY }}
- name: Create package (amd64)
run: |
go run build.go -goarch amd64 zip
env:
CGO_ENABLED: "1"
- name: Create package (arm64 cross)
run: |
cat <<EOT > xgo.sh
#!/bin/bash
CGO_ENABLED=1 \
CGO_CFLAGS="-target arm64-apple-macos10.15" \
CGO_LDFLAGS="-target arm64-apple-macos10.15" \
go "\$@"
EOT
chmod 755 xgo.sh
go run build.go -gocmd ./xgo.sh -goarch arm64 zip
env:
CGO_ENABLED: "1"
- name: Create package (universal)
run: |
rm -rf _tmp
mkdir _tmp
pushd _tmp
unzip ../syncthing-macos-amd64-*.zip
unzip ../syncthing-macos-arm64-*.zip
lipo -create syncthing-macos-amd64-*/syncthing syncthing-macos-arm64-*/syncthing -o syncthing
amd64=(syncthing-macos-amd64-*)
universal="${amd64/amd64/universal}"
mv "$amd64" "$universal"
mv syncthing "$universal"
zip -r "../$universal.zip" "$universal"
- name: Archive artifacts
uses: actions/upload-artifact@v4
with:
name: packages-macos
path: syncthing-*.zip
notarize-macos:
name: Notarize for macOS
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
environment: signing
needs:
- package-macos
- basics
runs-on: macos-latest
steps:
- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: packages-macos
- name: Notarize binaries
run: |
APPSTORECONNECT_API_KEY_PATH="$RUNNER_TEMP/apikey.p8"
echo "$APPSTORECONNECT_API_KEY" | base64 -d -o "$APPSTORECONNECT_API_KEY_PATH"
for file in syncthing-macos-*.zip ; do
xcrun notarytool submit \
-k "$APPSTORECONNECT_API_KEY_PATH" \
-d "$APPSTORECONNECT_API_KEY_ID" \
-i "$APPSTORECONNECT_API_KEY_ISSUER" \
$file
done
env:
APPSTORECONNECT_API_KEY: ${{ secrets.APPSTORECONNECT_API_KEY }}
APPSTORECONNECT_API_KEY_ID: ${{ secrets.APPSTORECONNECT_API_KEY_ID }}
APPSTORECONNECT_API_KEY_ISSUER: ${{ secrets.APPSTORECONNECT_API_KEY_ISSUER }}
#
# Cross compile other unixes
#
package-cross:
name: Package cross compiled
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: false
check-latest: true
- name: Get actual Go version
run: |
go version
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
- uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-cross-${{ hashFiles('**/go.sum') }}
- name: Create packages
run: |
platforms=$(go tool dist list \
| grep -v aix/ppc64 \
| grep -v android/ \
| grep -v darwin/ \
| grep -v ios/ \
| grep -v js/ \
| grep -v linux/ \
| grep -v nacl/ \
| grep -v plan9/ \
| grep -v windows/ \
| grep -v /wasm \
)
# Build for each platform with errors silenced, because we expect
# some oddball platforms to fail. This avoids a bunch of errors in
# the GitHub Actions output, instead summarizing each build
# failure as a warning.
for plat in $platforms; do
goos="${plat%/*}"
goarch="${plat#*/}"
echo "::group ::$plat"
if ! go run build.go -goos "$goos" -goarch "$goarch" tar 2>/dev/null; then
echo "::warning ::Failed to build for $plat"
fi
echo "::endgroup::"
done
env:
CGO_ENABLED: "0"
- name: Archive artifacts
uses: actions/upload-artifact@v4
with:
name: packages-other
path: syncthing-*.tar.gz
#
# Source
#
package-source:
name: Package source code
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: false
check-latest: true
- name: Package source
run: |
version=$(go run build.go version)
echo "$version" > RELEASE
go mod vendor
go run build.go assets
cd ..
tar c -z -f "syncthing-source-$version.tar.gz" \
--exclude .git \
syncthing
mv "syncthing-source-$version.tar.gz" syncthing
- name: Archive artifacts
uses: actions/upload-artifact@v4
with:
name: packages-source
path: syncthing-source-*.tar.gz
#
# Sign binaries for auto upgrade, generate ASC signature files
#
sign-for-upgrade:
name: Sign for upgrade
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
environment: signing
needs:
- basics
- package-windows
- package-linux
- package-macos
- package-cross
- package-source
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/checkout@v4
with:
repository: syncthing/release-tools
path: tools
fetch-depth: 0
- name: Download artifacts
uses: actions/download-artifact@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: false
check-latest: true
- name: Install signing tool
run: |
go install ./cmd/stsigtool
- name: Sign archives
run: |
export PRIVATE_KEY="$RUNNER_TEMP/privkey.pem"
export PATH="$PATH:$(go env GOPATH)/bin"
echo "$STSIGTOOL_PRIVATE_KEY" | base64 -d > "$PRIVATE_KEY"
mkdir packages
mv packages-*/* packages
pushd packages
"$GITHUB_WORKSPACE/tools/sign-only"
rm -f "$PRIVATE_KEY"
env:
STSIGTOOL_PRIVATE_KEY: ${{ secrets.STSIGTOOL_PRIVATE_KEY }}
- name: Create and sign .asc files
run: |
sudo apt update
sudo apt -y install gnupg
export SIGNING_KEY="$RUNNER_TEMP/gpg-secret.asc"
echo "$GNUPG_SIGNING_KEY_BASE64" | base64 -d > "$SIGNING_KEY"
gpg --import < "$SIGNING_KEY"
pushd packages
files=(*.tar.gz *.zip)
sha1sum "${files[@]}" | gpg --clearsign > sha1sum.txt.asc
sha256sum "${files[@]}" | gpg --clearsign > sha256sum.txt.asc
gpg --sign --armour --detach syncthing-source-*.tar.gz
popd
rm -f "$SIGNING_KEY" .gnupg
env:
GNUPG_SIGNING_KEY_BASE64: ${{ secrets.GNUPG_SIGNING_KEY_BASE64 }}
- name: Archive artifacts
uses: actions/upload-artifact@v4
with:
name: packages-signed
path: packages/*
#
# Debian
#
package-debian:
name: Package for Debian
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: false
check-latest: true
- name: Get actual Go version
run: |
go version
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
- uses: ruby/setup-ruby@v1
with:
ruby-version: '3.0'
- name: Install fpm
run: |
gem install fpm
- uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-debian-${{ hashFiles('**/go.sum') }}
- name: Package for Debian
run: |
for arch in amd64 i386 armhf armel arm64 ; do
go run build.go -no-upgrade -installsuffix=no-upgrade -goarch "$arch" deb
done
env:
BUILD_USER: debian
- name: Archive artifacts
uses: actions/upload-artifact@v4
with:
name: debian-packages
path: "*.deb"
#
# Nightlies
#
publish-nightly:
name: Publish nightly build
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && startsWith(github.ref, 'refs/heads/release-nightly')
environment: signing
needs:
- sign-for-upgrade
- notarize-macos
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
repository: syncthing/release-tools
path: tools
fetch-depth: 0
- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: packages-signed
path: packages
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: false
check-latest: true
- name: Create release json
run: |
cd packages
"$GITHUB_WORKSPACE/tools/generate-release-json" "$BASE_URL" > nightly.json
env:
BASE_URL: https://syncthing.ams3.digitaloceanspaces.com/nightly/
- name: Push artifacts
uses: docker://docker.io/rclone/rclone:latest
env:
RCLONE_CONFIG_SPACES_TYPE: s3
RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
RCLONE_CONFIG_SPACES_ACL: public-read
with:
args: sync packages spaces:syncthing/nightly
#
# Push release artifacts to Spaces
#
publish-release-files:
name: Publish release files
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/release'
environment: signing
needs:
- sign-for-upgrade
- package-debian
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Download signed packages
uses: actions/download-artifact@v4
with:
name: packages-signed
path: packages
- name: Download debian packages
uses: actions/download-artifact@v4
with:
name: debian-packages
path: packages
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: false
check-latest: true
- name: Set version
run: |
version=$(go run build.go version)
echo "VERSION=$version" >> $GITHUB_ENV
- name: Push to Spaces (${{ env.VERSION }})
uses: docker://docker.io/rclone/rclone:latest
env:
RCLONE_CONFIG_SPACES_TYPE: s3
RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
RCLONE_CONFIG_SPACES_ACL: public-read
with:
args: sync packages spaces:syncthing/release/${{ env.VERSION }}
- name: Push to Spaces (latest)
uses: docker://docker.io/rclone/rclone:latest
env:
RCLONE_CONFIG_SPACES_TYPE: s3
RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
RCLONE_CONFIG_SPACES_ACL: public-read
with:
args: sync spaces:syncthing/release/${{ env.VERSION }} spaces:syncthing/release/latest
#
# Build and push to Docker Hub
#
docker-syncthing:
name: Build and push Docker images
runs-on: ubuntu-latest
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/infrastructure' || startsWith(github.ref, 'refs/heads/release-'))
environment: docker
strategy:
matrix:
pkg:
- syncthing
- strelaysrv
- stdiscosrv
include:
- pkg: syncthing
dockerfile: Dockerfile
image: syncthing/syncthing
- pkg: strelaysrv
dockerfile: Dockerfile.strelaysrv
image: syncthing/relaysrv
- pkg: stdiscosrv
dockerfile: Dockerfile.stdiscosrv
image: syncthing/discosrv
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: false
check-latest: true
- name: Get actual Go version
run: |
go version
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
- uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-docker-${{ matrix.pkg }}-${{ hashFiles('**/go.sum') }}
- name: Build binaries
run: |
for arch in amd64 arm64 arm; do
go run build.go -goos linux -goarch "$arch" -no-upgrade build ${{ matrix.pkg }}
mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-"$arch"
done
env:
CGO_ENABLED: "0"
BUILD_USER: docker
- name: Check if we will be able to push images
run: |
if [[ "${{ secrets.DOCKERHUB_TOKEN }}" != "" ]]; then
echo "DOCKER_PUSH=true" >> $GITHUB_ENV;
fi
- name: Login to Docker Hub
uses: docker/login-action@v3
if: env.DOCKER_PUSH == 'true'
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Set version tags
run: |
version=$(go run build.go version)
version=${version#v}
if [[ $version == @([0-9]|[0-9][0-9]).@([0-9]|[0-9][0-9]).@([0-9]|[0-9][0-9]) ]] ; then
echo Release version, pushing to :latest and version tags
major=${version%.*.*}
minor=${version%.*}
tags=${{ matrix.image }}:$version,${{ matrix.image }}:$major,${{ matrix.image }}:$minor,${{ matrix.image }}:latest
elif [[ $version == *-rc.@([0-9]|[0-9][0-9]) ]] ; then
echo Release candidate, pushing to :rc
tags=${{ matrix.image }}:rc
else
echo Development version, pushing to :edge
tags=${{ matrix.image }}:edge
fi
echo "DOCKER_TAGS=$tags" >> $GITHUB_ENV
echo "VERSION=$version" >> $GITHUB_ENV
- name: Build and push Docker image
uses: docker/build-push-action@v5
with:
context: .
file: ${{ matrix.dockerfile }}
platforms: linux/amd64,linux/arm64,linux/arm/7
push: ${{ env.DOCKER_PUSH == 'true' }}
tags: ${{ env.DOCKER_TAGS }}
labels: |
org.opencontainers.image.version=${{ env.VERSION }}
org.opencontainers.image.revision=${{ github.sha }}
#
# Check for known vulnerabilities in Go dependencies
#
govulncheck:
runs-on: ubuntu-latest
name: Run govulncheck
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache: false
check-latest: true
- name: run govulncheck
run: |
go run build.go assets
go install golang.org/x/vuln/cmd/govulncheck@latest
govulncheck ./...

View File

@@ -1,21 +0,0 @@
name: Trigger nightly build & release
on:
workflow_dispatch:
schedule:
# Run nightly build at 01:00 UTC
- cron: '00 01 * * *'
jobs:
trigger-nightly:
runs-on: ubuntu-latest
name: Push to release-nightly to trigger build
steps:
- uses: actions/checkout@v4
with:
token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
fetch-depth: 0
- run: |
git push origin main:release-nightly

View File

@@ -1,28 +0,0 @@
name: Update translations and documentation
on:
workflow_dispatch:
schedule:
- cron: '42 3 * * 1'
jobs:
update_transifex_docs:
runs-on: ubuntu-latest
name: Update translations and documentation
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
- uses: actions/setup-go@v5
with:
go-version: stable
- run: |
set -euo pipefail
git config --global user.name 'Syncthing Release Automation'
git config --global user.email 'release@syncthing.net'
bash build.sh translate
bash build.sh prerelease
git push
env:
WEBLATE_TOKEN: ${{ secrets.WEBLATE_TOKEN }}

9
.gitignore vendored
View File

@@ -1,10 +1,11 @@
/syncthing
/stdiscosrv
syncthing.exe
stdiscosrv.exe
*.tar.gz
*.zip
*.asc
*.deb
*.exe
.jshintrc
coverage.out
files/pidx
@@ -14,7 +15,5 @@ coverage.xml
syncthing.sig
RELEASE
deb
*.bz2
/repos
/proto/scripts/protoc-gen-gosyncthing
/gui/next-gen-gui
lib/auto/gui.files.go
snapcraft.yaml

View File

@@ -1,26 +0,0 @@
linters-settings:
maligned:
suggest-new: true
linters:
enable-all: true
disable:
- goimports
- depguard
- lll
- gochecknoinits
- gochecknoglobals
- gofmt
- scopelint
- gocyclo
- funlen
- wsl
- gocognit
- godox
service:
golangci-lint-version: 1.21.x
prepare:
- rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
- GO111MODULE=on go mod vendor
- go run build.go assets

1
.mailmap Symbolic link
View File

@@ -0,0 +1 @@
NICKS

View File

@@ -1,4 +0,0 @@
line_ending: lf
formatter:
type: basic
retain_line_breaks: true

247
AUTHORS
View File

@@ -1,347 +1,120 @@
# This is the official list of Syncthing authors for copyright purposes.
#
# THIS FILE IS MOSTLY AUTO GENERATED. IF YOU'VE MADE A COMMIT TO THE
# REPOSITORY YOU WILL BE ADDED HERE AUTOMATICALLY WITHOUT THE NEED FOR
# ANY MANUAL ACTION.
#
# That said, you are welcome to correct your name or add a nickname / GitHub
# user name as appropriate. The format is:
# The format is:
#
# Name Name Name (nickname) <email1@example.com> <email2@example.com>
#
# The in-GUI authors list is periodically automatically updated from the
# contents of this file.
#
# The NICKS list is auto generated from this file.
Aaron Bieber (qbit) <qbit@deftly.net>
Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com> <adam@proactiveservices.co.uk>
Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com>
Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
Alan Pope <alan@popey.com>
Alberto Donato <albertodonato@users.noreply.github.com>
Aleksey Vasenev <margtu-fivt@ya.ru>
Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
Alex Lindeman <139387+aelindeman@users.noreply.github.com>
Alex Xu <alex.hello71@gmail.com>
Alexander Graf (alex2108) <register-github@alex-graf.de>
Alexander Seiler <seileralex@gmail.com>
Alexandre Alves <alexandrealvesdb.contact@gmail.com>
Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
Aman Gupta <aman@tmm1.net>
Anatoli Babenia <anatoli@rainforce.org>
Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
Andreas Sommer <andreas.sommer87@googlemail.com>
andresvia <andres.via@gmail.com>
Andrew Dunham (andrew-d) <andrew@du.nham.ca>
Andrew Meyer <andrewm.bpi@gmail.com>
Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
andyleap <andyleap@gmail.com>
Anjan Momi <anjan@momi.ca>
Anthony Goeckner <agoeckner@users.noreply.github.com>
Andrey D (scienmind) <scintertech@cryptolab.net>
Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
Antony Male (canton7) <antony.male@gmail.com>
Anur <anurnomeru@163.com>
Aranjedeath <Aranjedeath@users.noreply.github.com>
Arkadiusz Tymiński <gevleeog@gmail.com>
Aroun <login@b-vo.fr>
Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
Artur Zubilewicz <AkaZecik@users.noreply.github.com>
Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
Aurélien Rainone <476650+arl@users.noreply.github.com>
BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com>
Bart De Vries (mogwa1) <devriesb@gmail.com>
Beat Reichenbach <44111292+beatreichenbach@users.noreply.github.com>
Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
Ben Shepherd (benshep) <bjashepherd@gmail.com>
Ben Sidhom (bsidhom) <bsidhom@gmail.com>
Benedikt Heine (bebehei) <bebe@bebehei.de>
Benedikt Morbach <benedikt.morbach@googlemail.com>
Benjamin Nater <17193640+bn4t@users.noreply.github.com>
Benno Fünfstück <benno.fuenfstueck@gmail.com>
Benny Ng (tpng) <benny.tpng@gmail.com>
boomsquared <54829195+boomsquared@users.noreply.github.com>
Boqin Qin <bobbqqin@bupt.edu.cn>
Boris Rybalkin <ribalkin@gmail.com>
Brandon Philips (philips) <brandon@ifup.org>
Brendan Long (brendanlong) <self@brendanlong.com>
Brian R. Becker (brbecker) <brbecker@gmail.com>
bt90 <btom1990@googlemail.com>
Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
Catfriend1 <16361913+Catfriend1@users.noreply.github.com>
Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
Carsten Hagemann (Moter8) <moter8@gmail.com>
Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
Cedric Staniewski (xduugu) <cedric@gmx.ca>
chenrui <rui@meetup.com>
Chih-Hsuan Yen <yan12125@gmail.com> <1937689+yan12125@users.noreply.github.com>
Choongkyu <choongkyu.kim+gh@gmail.com> <vapidlyrapid+gh@gmail.com>
Chris Howie (cdhowie) <me@chrishowie.com>
Chris Joel (cdata) <chris@scriptolo.gy>
Chris Tonkinson <chris@masterbran.ch>
Christian Kujau <ckujau@users.noreply.github.com>
Christian Prescott <me@christianprescott.com>
chucic <chucic@seznam.cz>
cjc7373 <niuchangcun@gmail.com>
Colin Kennedy (moshen) <moshen.colin@gmail.com>
Cromefire_ <tim.l@nghorst.net> <26320625+cromefire@users.noreply.github.com>
cui fliter <imcusg@gmail.com>
Cyprien Devillez <cypx@users.noreply.github.com>
d-volution <49024624+d-volution@users.noreply.github.com>
Dale Visser <dale.visser@live.com>
Dan <benda.daniel@gmail.com>
Daniel Barczyk <46358936+DanielBarczyk@users.noreply.github.com>
Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
Daniel Martí (mvdan) <mvdan@mvdan.cc>
Daniel Padrta <64928366+danpadcz@users.noreply.github.com>
Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com>
DeflateAwning <11021263+DeflateAwning@users.noreply.github.com>
Denis A. (dva) <denisva@gmail.com>
Dennis Wilson (snnd) <dw@risu.io>
dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com> <27856297+dependabot-preview[bot]@users.noreply.github.com>
dependabot[bot] <dependabot[bot]@users.noreply.github.com> <49699333+dependabot[bot]@users.noreply.github.com>
derekriemer <derek.riemer@colorado.edu>
DerRockWolf <50499906+DerRockWolf@users.noreply.github.com>
desbma <desbma@users.noreply.github.com>
Devon G. Redekopp <devon@redekopp.com>
diemade <spamkill@posteo.ch>
digital <didev@dinid.net>
Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com>
Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
Domenic Horner <domenic@tgxn.net>
Dominik Heidler (asdil12) <dominik@heidler.eu>
Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
Elliot Huffman <thelich2@gmail.com>
Emil Hessman (ceh) <emil@hessman.se>
Emil Lundberg <emil@emlun.se>
Eng Zer Jun <engzerjun@gmail.com>
entity0xfe <109791748+entity0xfe@users.noreply.github.com> <entity0xfe@my.domain>
Eric Lesiuta <elesiuta@gmail.com>
Eric P <eric@kastelo.net>
Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
Evan Spensley <94762716+0evan@users.noreply.github.com>
Evgeny Kuznetsov <evgeny@kuznetsov.md>
Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
Felix <53702818+f-eliks@users.noreply.github.com>
Felix Ableitner (Nutomic) <me@nutomic.com>
Felix Lampe <mail@flampe.de>
Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
Frank Isemann (fti7) <frank@isemann.name>
Gahl Saraf <saraf.gahl@gmail.com> <gahl@raftt.io>
georgespatton <georgespatton@users.noreply.github.com>
ghjklw <malo@jaffre.info>
Gilli Sigurdsson (gillisig) <gilli@vx.is>
Gleb Sinyavskiy <zhulik.gleb@gmail.com>
Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
greatroar <61184462+greatroar@users.noreply.github.com>
Greg <gco@jazzhaiku.com>
guangwu <guoguangwu@magic-shield.com>
gudvinr <gudvinr@gmail.com>
Han Boetes <han@boetes.org>
HansK-p <42314815+HansK-p@users.noreply.github.com>
Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
Heiko Zuerker (Smiley73) <heiko@zuerker.org>
Hugo Locurcio <hugo.locurcio@hugo.pro>
Iain Barnett <iainspeed@gmail.com>
Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
ignacy123 <ignacy.buczek@onet.pl>
Ikko Ashimine <eltociear@gmail.com>
Ilya Brin <464157+ilyabrin@users.noreply.github.com>
Iskander Sharipov (Alex) <quasilyte@gmail.com>
Jaakko Hannikainen (jgke) <jgke@jgke.fi>
Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
Jack Croft <jccroft1@users.noreply.github.com>
Jacob <jyundt@gmail.com>
Jake Peterson (acogdev) <jake@acogdev.com>
Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net> <jborg@coreweave.com>
James O'Beirne <wild-github@au92.org>
Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
janost <janost@tuta.io>
Jaroslav Lichtblau <svetlemodry@users.noreply.github.com>
Jaroslav Malec (dzarda) <dzardacz@gmail.com>
jaseg <githubaccount@jaseg.net>
Jaspitta <ste.scarpitta@gmail.com>
Jauder Ho <jauderho@users.noreply.github.com>
Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
Jaya Kumar <jaya.kumar@ict.nl>
Jeffery To <jeffery.to@gmail.com>
jelle van der Waa <jelle@vdwaa.nl>
Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
Jesse Lucas <jesse@jesselucas.com>
Jochen Voss (seehuhn) <voss@seehuhn.de>
Johan Andersson <j@i19.se>
Johan Vromans (sciurius) <jvromans@squirrel.nl>
John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
Jonas Thelemann <e-mail@jonas-thelemann.de>
Jonathan <artback@protonmail.com> <jonagn@gmail.com>
Jonathan Cross <jcross@gmail.com>
Jonta <359397+Jonta@users.noreply.github.com>
Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
jtagcat <git-514635f7@jtag.cat> <git-12dbd862@jtag.cat>
Julian Lehrhuber <jul13579@users.noreply.github.com>
Jörg Thalheim <Mic92@users.noreply.github.com>
Jędrzej Kula <kula.jedrek@gmail.com>
K.B.Dharun Krishna <kbdharunkrishna@gmail.com>
Kalle Laine <pahakalle@protonmail.com>
Karol Różycki (krozycki) <rozycki.karol@gmail.com>
Kebin Liu <lkebin@gmail.com>
Keith Harrison <keithh@protonmail.com>
Keith Turner <kturner@apache.org>
Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
Kevin Allen (ironmig) <kma1660@gmail.com>
Kevin Bushiri (keevBush) <keevbush@gmail.com> <36192217+keevBush@users.noreply.github.com>
Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
klemens <ka7@github.com>
Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
kylosus <33132401+kylosus@users.noreply.github.com>
Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
Lars Lehtonen <lars.lehtonen@gmail.com>
Laurent Arnoud <laurent@spkdev.net>
Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
Leo Arias (elopio) <yo@elopio.net>
Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
Lode Hoste (Zillode) <zillode@zillode.be>
Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
LSmithx2 <42276854+lsmithx2@users.noreply.github.com>
Lukas Lihotzki <lukas@lihotzki.de>
Luke Hamburg <1992842+luckman212@users.noreply.github.com>
luzpaz <luzpaz@users.noreply.github.com>
Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
Marc Pujol (kilburn) <kilburn@la3.org>
Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
marco-m <marco.molteni@laposte.net>
Marcus Legendre <marcus.legendre@gmail.com>
Mario Majila <mariustshipichik@gmail.com>
Mark Pulford (mpx) <mark@kyne.com.au>
Martchus <martchus@gmx.net>
Martin Polehla <p0l0us@users.noreply.github.com>
Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
Mateusz Ż <thedead4fun@live.com>
Matic Potočnik <hairyfotr@gmail.com>
Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
Matt Robenolt <matt@ydekproductions.com>
Matteo Ruina <matteo.ruina@gmail.com>
Maurizio Tomasi <ziotom78@gmail.com>
Max <github@germancoding.com>
Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
MaximAL <almaximal@ya.ru>
Maxime Thirouin <m@moox.io>
Maximilian <maxi.rostock@outlook.de> <public@complexvector.space>
mclang <1721600+mclang@users.noreply.github.com>
Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
Michael Ploujnikov (plouj) <ploujj@gmail.com>
Michael Rienstra <mrienstra@gmail.com>
Michael Tilli (pyfisch) <pyfisch@gmail.com>
MichaIng <micha@dietpi.com>
Migelo <miha@filetki.si>
Mike Boone <mike@boonedocks.net>
MikeLund <MikeLund@users.noreply.github.com>
MikolajTwarog <43782609+MikolajTwarog@users.noreply.github.com>
Mingxuan Lin <gdlmx@users.noreply.github.com>
mv1005 <49659413+mv1005@users.noreply.github.com>
Nate Morrison (nrm21) <natemorrison@gmail.com>
Naveen <172697+naveensrinivasan@users.noreply.github.com>
nf <nf@wh3rd.net>
Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
Nick Busey <NickBusey@users.noreply.github.com>
Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
Nicolas Perraut <n.perraut@gmail.com>
Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
Nils Jakobi (thunderstorm99) <jakobi.nils@gmail.com>
NinoM4ster <ninom4ster@gmail.com>
Nitroretro <43112364+Nitroretro@users.noreply.github.com>
NoLooseEnds <jon.koslung@gmail.com>
Oliver Freyermuth <o.freyermuth@googlemail.com>
orangekame3 <miya.org.0309@gmail.com>
otbutz <tbutz@optitool.de>
Otiel <Otiel@users.noreply.github.com>
overkill <22098433+0verk1ll@users.noreply.github.com>
Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
Pablo <pbaeyens31+github@gmail.com>
Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
Paul Brit <paulbrit44@gmail.com>
Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
Paweł Rozlach <vespian@users.noreply.github.com>
perewa <cavalcante.ten@gmail.com>
Peter Badida <KeyWeeUsr@users.noreply.github.com>
Peter Dave Hello <hsu@peterdavehello.org>
Peter Hoeg (peterhoeg) <peter@speartail.com>
Peter Marquardt (wwwutz) <wwwutz@gmail.com> <wwwutz@googlemail.com>
Phani Rithvij <phanirithvij2000@gmail.com>
Phil Davis <phil.davis@inf.org>
Philippe Schommers (filoozoom) <philippe@schommers.be>
Phill Luby (pluby) <phill.luby@newredo.com>
Pier Paolo Ramon <ramonpierre@gmail.com>
Piotr Bejda (piobpl) <piotrb10@gmail.com>
Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
Quentin Hibon <qh.public@yahoo.com>
Rahmi Pruitt <rjpruitt16@gmail.com>
red_led <red-led@users.noreply.github.com>
Richard Hartmann <RichiH@users.noreply.github.com>
Robert Carosi (nov1n) <robert@carosi.nl>
Roberto Santalla <roobre@users.noreply.github.com>
Robin Schoonover <robin@cornhooves.org>
Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
Ross Smith II (rasa) <ross@smithii.com>
rubenbe <github-com-00ff86@vandamme.email>
Ruslan Yevdokymov <38809160+ruslanye@users.noreply.github.com>
Ryan Qian <i@bitbili.net>
Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
Scott Klupfel (kluppy) <kluppy@going2blue.com>
sec65 <106604020+sec65@users.noreply.github.com>
Sergey Mishin (ralder) <ralder@yandex.ru>
Sertonix <83883937+Sertonix@users.noreply.github.com>
Shaarad Dalvi <60266155+shaaraddalvi@users.noreply.github.com> <shdalv@microsoft.com>
Simon Frei (imsodin) <freisim93@gmail.com>
Simon Mwepu <simonmwepu@gmail.com>
Sly_tom_cat <slytomcat@mail.ru>
Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
Steven Eckhoff <steven.eckhoff.opensource@gmail.com>
Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
Sven Bachmann <dev@mcbachmann.de>
Syncthing Automation <automation@syncthing.net>
Syncthing Release Automation <release@syncthing.net>
Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
Thomas <9749173+uhthomas@users.noreply.github.com>
Thomas Hipp <thomashipp@gmail.com>
Tim Abell (timabell) <tim@timwise.co.uk>
Tim Howes (timhowes) <timhowes@berkeley.edu>
Tim Nordenfur <tim@gurka.se>
Tobias Klauser <tobias.klauser@gmail.com>
Tobias Nygren (tnn2) <tnn@nygren.pp.se>
Tobias Tom (tobiastom) <t.tom@succont.de>
Tom Jakubowski <tom@crystae.net>
Tomasz Wilczyński <5626656+tomasz1986@users.noreply.github.com> <twilczynski@naver.com>
Tommy Thorn <tommy-github-email@thorn.ws>
Tomas Cerveny (kozec) <kozec@kozec.com>
Tully Robinson (tojrobinson) <tully@tojr.org>
Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
Tyler Kropp <kropptyler@gmail.com>
Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
vapatel2 <149737089+vapatel2@users.noreply.github.com>
Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
Victor Buinsky (buinsky) <vix_booja@tut.by>
Vik <63919734+ViktorOn@users.noreply.github.com>
Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
villekalliomaki <53118179+villekalliomaki@users.noreply.github.com>
Vladimir Rusinov <vrusinov@google.com> <vladimir.rusinov@gmail.com>
wangguoliang <liangcszzu@163.com>
Will Rouesnel <wrouesnel@wrouesnel.com>
William A. Kennington III (wkennington) <william@wkennington.com>
wouter bolsterlee <wouter@bolsterl.ee>
Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
xarx00 <xarx00@users.noreply.github.com>
Xavier O. (damajor) <damajor@gmail.com>
xjtdy888 (xjtdy888) <xjtdy888@163.com>
Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
佛跳墙 <daoquan@qq.com>
落心 <luoxin.ttt@gmail.com>

View File

@@ -1,73 +1,92 @@
# Contributor Covenant Code of Conduct
## Conduct
## Our Pledge
* We are committed to providing a friendly, safe and welcoming
environment for all, regardless of gender, sexual orientation,
disability, ethnicity, religion, or similar personal characteristic.
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, gender identity and expression, level of experience,
education, socio-economic status, nationality, personal appearance, race,
religion, or sexual identity and orientation.
* On IRC, please avoid using overtly sexual nicknames or other nicknames
that might detract from a friendly, safe and welcoming environment for
all.
## Our Standards
* Please be kind and courteous. There's no need to be mean or rude.
Examples of behavior that contributes to creating a positive environment
include:
* Respect that people have differences of opinion and that every design
or implementation choice carries a trade-off and numerous costs. There
is seldom a right answer.
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
* Please keep unstructured critique to a minimum. If you have solid
ideas you want to experiment with, make a fork and see how it works.
Examples of unacceptable behavior by participants include:
* We will exclude you from interaction if you insult, demean or harass
anyone. That is not welcome behaviour. We interpret the term
"harassment" as including the definition in the <a
href="http://citizencodeofconduct.org/">Citizen Code of Conduct</a>;
if you have any lack of clarity about what might be included in that
concept, please read their definition. In particular, we don't
tolerate behavior that excludes people in socially marginalized
groups.
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
* Private harassment is also unacceptable. No matter who you are, if you
feel you have been or are being harassed or made uncomfortable by a
community member, please contact one of the channel ops or any of the
Syncthing core team immediately. Whether you're a regular contributor
or a newcomer, we care about making this community a safe place for
you and we've got your back.
## Our Responsibilities
* Likewise any spamming, trolling, flaming, baiting or other
attention-stealing behaviour is not welcome.
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
## Moderation
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
These are the policies for upholding our community's standards of
conduct in our communication channels, most notably in Syncthing-related
IRC channels and on the web forum.
## Scope
1. Remarks that violate the Syncthing standards of conduct, including
hateful, hurtful, oppressive, or exclusionary remarks, are not
allowed. (Cursing is allowed, but never targeting another user, and
never in a hateful manner.)
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
2. Remarks that moderators find inappropriate, whether listed in the
code of conduct or not, are also not allowed.
## Enforcement
3. Moderators will first respond to such remarks with a warning.
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at security@syncthing.net. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
4. If the warning is unheeded, the user will be "kicked," i.e., kicked
out of the communication channel to cool off.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
5. If the user comes back and continues to make trouble, they will be
banned, i.e., indefinitely excluded.
## Attribution
6. Moderators may choose at their discretion to un-ban the user if it
was a first offense and they offer the offended party a genuine
apology.
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
7. If a moderator bans someone and you think it was unjustified, please
take it up with that moderator, or with a different moderator, **in
private**. Complaints about bans in-channel are not allowed.
[homepage]: https://www.contributor-covenant.org
8. Moderators are held to a higher standard than other community
members. If a moderator creates an inappropriate situation, they
should expect less leeway than others.
In the Syncthing community we strive to go the extra step to look out
for each other. Don't just aim to be technically unimpeachable, try to
be your best self. In particular, avoid flirting with offensive or
sensitive issues, particularly if they're off-topic; this all too
often leads to unnecessary fights, hurt feelings, and damaged trust;
worse, it can drive people away from the community entirely.
And if someone takes issue with something you said or did, resist the
urge to be defensive. Just stop doing what it was they complained about
and apologize. Even if you feel you were misinterpreted or unfairly
accused, chances are good there was something you could've communicated
better — remember that it's your responsibility to make your fellow
community members comfortable. Everyone wants to get along and we are
all here first and foremost because we want to talk about cool
technology. You will find that people will be eager to assume good
intent and forgive as long as you earn their trust.
*Adapted from the [Rust Code of Conduct](https://github.com/rust-lang/rust/wiki/Note-development-policy#conduct)*
*Adapted from the [Node.js Policy on Trolling](http://blog.izs.me/post/30036893703/policy-on-trolling)*

View File

@@ -1,6 +1,6 @@
## Reporting Bugs
Please file bugs in the [GitHub Issue
Please file bugs in the [Github Issue
Tracker](https://github.com/syncthing/syncthing/issues). Include at
least the following:
@@ -24,15 +24,10 @@ too much information will never get you yelled at. :)
## Contributing Translations
All translations are done via
[Weblate](https://hosted.weblate.org/projects/syncthing/). If you wish
to contribute to a translation, just head over there and sign up.
[Transifex](https://www.transifex.com/projects/p/syncthing/). If you
wish to contribute to a translation, just head over there and sign up.
Before every release, the language resources are updated from the
latest info on Weblate.
Note that the previously used service at
[Transifex](https://www.transifex.com/projects/p/syncthing/) is being
retired and we kindly ask you to sign up on Weblate for continued
involvement.
latest info on Transifex.
## Contributing Code

View File

@@ -1,57 +0,0 @@
ARG GOVERSION=latest
#
# Maybe build Syncthing. This is a bit ugly as we can't make an entire
# section of the Dockerfile conditional, so we end up always pulling the
# golang image as builder. Then we check if the executable we need already
# exists (pre-built) otherwise we build it.
#
FROM golang:$GOVERSION AS builder
ARG BUILD_USER
ARG BUILD_HOST
ARG TARGETARCH
WORKDIR /src
COPY . .
ENV CGO_ENABLED=0
RUN if [ ! -f syncthing-linux-$TARGETARCH ] ; then \
go run build.go -no-upgrade build syncthing ; \
mv syncthing syncthing-linux-$TARGETARCH ; \
fi
#
# The rest of the Dockerfile uses the binary from the builder, prebuilt or
# not.
#
FROM alpine
ARG TARGETARCH
LABEL org.opencontainers.image.authors="The Syncthing Project" \
org.opencontainers.image.url="https://syncthing.net" \
org.opencontainers.image.documentation="https://docs.syncthing.net" \
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
org.opencontainers.image.vendor="The Syncthing Project" \
org.opencontainers.image.licenses="MPL-2.0" \
org.opencontainers.image.title="Syncthing"
EXPOSE 8384 22000/tcp 22000/udp 21027/udp
VOLUME ["/var/syncthing"]
RUN apk add --no-cache ca-certificates curl libcap su-exec tzdata
COPY --from=builder /src/syncthing-linux-$TARGETARCH /bin/syncthing
COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
ENV PUID=1000 PGID=1000 HOME=/var/syncthing
HEALTHCHECK --interval=1m --timeout=10s \
CMD curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -o --color=never OK || exit 1
ENV STGUIADDRESS=0.0.0.0:8384
ENV STHOMEDIR=/var/syncthing/config
RUN chmod 755 /bin/entrypoint.sh
ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing"]

View File

@@ -1,17 +0,0 @@
ARG GOVERSION=latest
FROM golang:$GOVERSION
LABEL org.opencontainers.image.authors="The Syncthing Project" \
org.opencontainers.image.url="https://syncthing.net" \
org.opencontainers.image.documentation="https://docs.syncthing.net" \
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
org.opencontainers.image.vendor="The Syncthing Project" \
org.opencontainers.image.licenses="MPL-2.0" \
org.opencontainers.image.title="Syncthing Builder"
# FPM to build Debian packages
RUN apt-get update && apt-get install -y --no-install-recommends \
locales rubygems ruby-dev build-essential git \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& gem install fpm

View File

@@ -1,16 +0,0 @@
FROM alpine
ARG TARGETARCH
LABEL org.opencontainers.image.authors="The Syncthing Project" \
org.opencontainers.image.url="https://syncthing.net" \
org.opencontainers.image.documentation="https://docs.syncthing.net" \
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
org.opencontainers.image.vendor="The Syncthing Project" \
org.opencontainers.image.licenses="MPL-2.0" \
org.opencontainers.image.title="Syncthing Crash Receiver"
EXPOSE 8080
COPY stcrashreceiver-linux-${TARGETARCH} /bin/stcrashreceiver
ENTRYPOINT [ "/bin/stcrashreceiver" ]

View File

@@ -1,42 +0,0 @@
ARG GOVERSION=latest
FROM golang:$GOVERSION AS builder
ARG BUILD_USER
ARG BUILD_HOST
ARG TARGETARCH
WORKDIR /src
COPY . .
ENV CGO_ENABLED=0
RUN if [ ! -f stdiscosrv-linux-$TARGETARCH ] ; then \
go run build.go -no-upgrade build stdiscosrv ; \
mv stdiscosrv stdiscosrv-linux-$TARGETARCH ; \
fi
FROM alpine
ARG TARGETARCH
LABEL org.opencontainers.image.authors="The Syncthing Project" \
org.opencontainers.image.url="https://syncthing.net" \
org.opencontainers.image.documentation="https://docs.syncthing.net" \
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
org.opencontainers.image.vendor="The Syncthing Project" \
org.opencontainers.image.licenses="MPL-2.0" \
org.opencontainers.image.title="Syncthing Discovery Server"
EXPOSE 19200 8443
VOLUME ["/var/stdiscosrv"]
RUN apk add --no-cache ca-certificates su-exec
COPY --from=builder /src/stdiscosrv-linux-$TARGETARCH /bin/stdiscosrv
COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
ENV PUID=1000 PGID=1000 HOME=/var/stdiscosrv
HEALTHCHECK --interval=1m --timeout=10s \
CMD nc -z localhost 8443 || exit 1
WORKDIR /var/stdiscosrv
ENTRYPOINT ["/bin/entrypoint.sh", "/bin/stdiscosrv"]

View File

@@ -1,24 +0,0 @@
FROM alpine
ARG TARGETARCH
LABEL org.opencontainers.image.authors="The Syncthing Project" \
org.opencontainers.image.url="https://syncthing.net" \
org.opencontainers.image.documentation="https://docs.syncthing.net" \
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
org.opencontainers.image.vendor="The Syncthing Project" \
org.opencontainers.image.licenses="MPL-2.0" \
org.opencontainers.image.title="Syncthing Relay Pool Server"
EXPOSE 8080
RUN apk add --no-cache ca-certificates su-exec curl
ENV PUID=1000 PGID=1000 MAXMIND_KEY=
RUN mkdir /var/strelaypoolsrv && chown 1000 /var/strelaypoolsrv
USER 1000
COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv
COPY script/strelaypoolsrv-entrypoint.sh /bin/entrypoint.sh
WORKDIR /var/strelaypoolsrv
ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaypoolsrv", "-listen", ":8080"]

View File

@@ -1,42 +0,0 @@
ARG GOVERSION=latest
FROM golang:$GOVERSION AS builder
ARG BUILD_USER
ARG BUILD_HOST
ARG TARGETARCH
WORKDIR /src
COPY . .
ENV CGO_ENABLED=0
RUN if [ ! -f strelaysrv-linux-$TARGETARCH ] ; then \
go run build.go -no-upgrade build strelaysrv ; \
mv strelaysrv strelaysrv-linux-$TARGETARCH ; \
fi
FROM alpine
ARG TARGETARCH
LABEL org.opencontainers.image.authors="The Syncthing Project" \
org.opencontainers.image.url="https://syncthing.net" \
org.opencontainers.image.documentation="https://docs.syncthing.net" \
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
org.opencontainers.image.vendor="The Syncthing Project" \
org.opencontainers.image.licenses="MPL-2.0" \
org.opencontainers.image.title="Syncthing Relay Server"
EXPOSE 22067 22070
VOLUME ["/var/strelaysrv"]
RUN apk add --no-cache ca-certificates su-exec
COPY --from=builder /src/strelaysrv-linux-$TARGETARCH /bin/strelaysrv
COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
ENV PUID=1000 PGID=1000 HOME=/var/strelaysrv
HEALTHCHECK --interval=1m --timeout=10s \
CMD nc -z localhost 22067 || exit 1
WORKDIR /var/strelaysrv
ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaysrv"]

View File

@@ -1,16 +0,0 @@
FROM alpine
ARG TARGETARCH
LABEL org.opencontainers.image.authors="The Syncthing Project" \
org.opencontainers.image.url="https://syncthing.net" \
org.opencontainers.image.documentation="https://docs.syncthing.net" \
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
org.opencontainers.image.vendor="The Syncthing Project" \
org.opencontainers.image.licenses="MPL-2.0" \
org.opencontainers.image.title="Syncthing Upgrades"
EXPOSE 8080
COPY stupgrades-linux-${TARGETARCH} /bin/stupgrades
ENTRYPOINT [ "/bin/stupgrades" ]

View File

@@ -1,16 +0,0 @@
FROM alpine
ARG TARGETARCH
LABEL org.opencontainers.image.authors="The Syncthing Project" \
org.opencontainers.image.url="https://syncthing.net" \
org.opencontainers.image.documentation="https://docs.syncthing.net" \
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
org.opencontainers.image.vendor="The Syncthing Project" \
org.opencontainers.image.licenses="MPL-2.0" \
org.opencontainers.image.title="Syncthing Usage Reporting Server"
EXPOSE 8080
COPY ursrv-linux-${TARGETARCH} /bin/ursrv
ENTRYPOINT [ "/bin/ursrv" ]

View File

@@ -24,19 +24,19 @@ to avoid corrupting the user's files.
### 2. Secure Against Attackers
Again, protecting the user's data is paramount. Regardless of our other
goals, we must never allow the user's data to be susceptible to eavesdropping
goals we must never allow the user's data to be susceptible to eavesdropping
or modification by unauthorized parties.
> This should be understood in context. It is not necessarily reasonable to
> expect Syncthing to be resistant against well equipped state level
> attackers. We will, however, do our best. Note also that this is different
> attackers. We will however do our best. Note also that this is different
> from anonymity which is not, currently, a goal.
### 3. Easy to Use
Syncthing should be approachable, understandable, and inclusive.
Syncthing should be approachable, understandable and inclusive.
> Complex concepts and maths form the base of Syncthing's functionality.
> Complex concepts and maths form the base of Synchting's functionality.
> This should nonetheless be abstracted or hidden to a degree where
> Syncthing is usable by the general public.
@@ -52,18 +52,18 @@ User interaction should be required only when absolutely necessary.
### 5. Universally Available
Syncthing should run on every common computer. We are mindful that the
latest technology is not always available to every individual.
latest technology is not always available to any given individual.
> Computers include desktops, laptops, servers, virtual machines, small
> general purpose computers such as Raspberry Pis and, *where possible*,
> tablets and phones. NAS appliances, toasters, cars, firearms, thermostats,
> and so on may include computing capabilities but it is not our goal for
> tablets and phones. NAS appliances, toasters, cars, firearms, thermostats
> and so on may include computing capabitilies but it is not our goal for
> Syncthing to run smoothly on these devices.
### 6. For Individuals
Syncthing is primarily about empowering the individual user with safe,
secure, and easy to use file synchronization.
secure and easy to use file synchronization.
> We acknowledge that it's also useful in an enterprise setting and include
> functionality to support that. If this is in conflict with the

28
ISSUE_TEMPLATE.md Normal file
View File

@@ -0,0 +1,28 @@
Do not report security issues in this bug tracker. Instead, contact
security@syncthing.net directly - see https://syncthing.net/security.html
for more information.
If your issue is a support request ("How do I get my devices to connect?"
or similar), please use the support forum at https://forum.syncthing.net/
where a large number of helpful people hang out. This issue tracker is for
reporting bugs or feature requests directly to the developers.
If your issue is a bug report, replace this boilerplate with a description
of the problem, being sure to include at least:
- what happened,
- what you expected to happen instead, and
- any steps to reproduce the problem.
Also fill out the version information below and add log output or
screenshots as appropriate.
If your issue is a feature request, simply replace this template text in
its entirety.
### Version Information
Syncthing Version: v0.x.y
OS Version: Windows 7 / Ubuntu 14.04 / ...
Browser Version: (if applicable, for GUI issues)

149
NICKS Normal file
View File

@@ -0,0 +1,149 @@
# This file maps email addresses used in commits to nicks used the changelog.
# It is auto generated from the AUTHORS file by script/authors.go.
0x010C <antoine.lamielle@0x010c.fr>
0x010C <gh@0x010c.fr>
acogdev <jake@acogdev.com>
adelq <aqalieh95@gmail.com>
adelq <adelq@users.noreply.github.com>
alessandro.g89 <alessandro.g89@gmail.com>
alex2108 <register-github@alex-graf.de>
andersonvom <andersonvom@gmail.com>
andrew-d <andrew@du.nham.ca>
asdil12 <dominik@heidler.eu>
AudriusButkevicius <audrius.butkevicius@gmail.com>
aviau <alexandre@alexandreviau.net>
aviau <aviau@debian.org>
bencurthoys <ben@bencurthoys.com>
benshep <bjashepherd@gmail.com>
bigbear2nd <bigbear2nd@gmail.com>
brbecker <brbecker@gmail.com>
brendanlong <self@brendanlong.com>
brgmnn <dan.arne.bergmann@gmail.com>
brgmnn <brgmnn@users.noreply.github.com>
bsidhom <bsidhom@gmail.com>
buinsky <vix_booja@tut.by>
burkemw3 <mburke@amplify.com>
burkemw3 <burkemw3@gmail.com>
calmh <jakob@nym.se>
calmh <jakob@kastelo.net>
canton7 <antony.male@gmail.com>
Cathryne <cathryne.linenweaver@gmail.com>
Cathryne <Cathryne@users.noreply.github.com>
cdata <chris@scriptolo.gy>
cdhowie <me@chrishowie.com>
ceh <emil@hessman.se>
cqcallaw <enlightened.despot@gmail.com>
damajor <damajor@gmail.com>
dinosore <dinosore@dbrsoftware.co.uk>
dtchanpura <dtchanpura@gmail.com>
dtchanpura <dcprime314@gmail.com>
dva <denisva@gmail.com>
dzarda <dzardacz@gmail.com>
eipiminus1 <eipiminusone+github@gmail.com>
eipiminus1 <eipiminus1@users.noreply.github.com>
elopio <yo@elopio.net>
facastagnini <federico.castagnini@gmail.com>
filoozoom <philippe@schommers.be>
frioux <frew@afoolishmanifesto.com>
frioux <frioux@gmail.com>
fti7 <frank@isemann.name>
gillisig <gilli@vx.is>
hadogenes <szafar@linux.pl>
imsodin <freisim93@gmail.com>
ironmig <kma1660@gmail.com>
jarlebring <jarlebring@gmail.com>
jayachithra <s.k.jayachithra@gmail.com>
jedie <github.com@jensdiemer.de>
jedie <git@jensdiemer.de>
jgke <jgke@jgke.fi>
jmdaweb <jmdaweb@hotmail.com>
jmdaweb <jmdaweb@users.noreply.github.com>
jpjp <jamespatterson@operamail.com>
jpjp <jpjp@users.noreply.github.com>
kamadak <kamada@nanohz.org>
KayoticSully <kayoticsully@gmail.com>
kc1212 <kc04bc@gmx.com>
kc1212 <kc1212@users.noreply.github.com>
kilburn <kilburn@la3.org>
kluppy <kluppy@going2blue.com>
kozec <kozec@kozec.com>
kralo <max.schulze@online.de>
kralo <kralo@users.noreply.github.com>
krozycki <rozycki.karol@gmail.com>
Kudalufi <kurt@va1der.ca>
Kudalufi <kurt.fitzner@gmail.com>
kwhite17 <kevinwhite1710@gmail.com>
letiemble <laurent.etiemble@gmail.com>
letiemble <laurent.etiemble@monobjc.net>
liusy182 <liusy182@gmail.com>
liusy182 <liusy182@hotmail.com>
lkwg82 <lkwg82@gmx.de>
LordLandon <lordlandon@gmail.com>
majedev <majed.alhajry@gmail.com>
marcindziadus <dziadus.marcin@gmail.com>
marclaporte <marc@marclaporte.com>
marclaporte <marc@laporte.name>
mateon1 <matin1111@wp.pl>
mogwa1 <devriesb@gmail.com>
moshen <moshen.colin@gmail.com>
Moter8 <moter8@gmail.com>
mpx <mark@kyne.com.au>
mvdan <mvdan@mvdan.cc>
Niller303 <nielsproest@hotmail.com>
Niller303 <seje.niels@hotmail.com>
norgeous <daniel@harte.me>
norgeous <daniel@danielharte.co.uk>
norgeous <norgeous@users.noreply.github.com>
nov1n <robert@carosi.nl>
nrm21 <natemorrison@gmail.com>
Nutomic <me@nutomic.com>
pascalj <github@pascalj.com>
pascalj <mail@pascal-jungblut.com>
peterhoeg <peter@speartail.com>
philips <brandon@ifup.org>
piobpl <piotrb10@gmail.com>
plouj <ploujj@gmail.com>
pluby <phill.luby@newredo.com>
ProactiveServices <aD@simplypeachy.co.uk>
ProactiveServices <simplypeachy@users.noreply.github.com>
ProactiveServices <ProactiveServices@users.noreply.github.com>
pyfisch <pyfisch@gmail.com>
qbit <qbit@deftly.net>
ralder <ralder@yandex.ru>
rasa <ross@smithii.com>
Rewt0r <rewt0r@gmx.com>
Rewt0r <Rewt0r@users.noreply.github.com>
rumpelsepp <stefan@sevenbyte.org>
rumpelsepp <rumpelsepp@sevenbyte.org>
sacheendra <sacheendra.t@gmail.com>
scienmind <scintertech@cryptolab.net>
sciurius <jvromans@squirrel.nl>
seehuhn <voss@seehuhn.de>
Smiley73 <heiko@zuerker.org>
snnd <dw@risu.io>
snugghash <suhas.gundimeda@gmail.com>
snugghash <snugghash@gmail.com>
Stefan-Code <stefan.github@gmail.com>
Stefan-Code <Stefan.github@gmail.com>
timabell <tim@timwise.co.uk>
timhowes <timhowes@berkeley.edu>
tnn2 <tnn@nygren.pp.se>
tojrobinson <tully@tojr.org>
tpng <benny.tpng@gmail.com>
tylerbrazier <tyler@tylerbrazier.com>
Unrud <unrud@openaliasbox.org>
Unrud <Unrud@users.noreply.github.com>
uok <ueomkail@gmail.com>
uok <uok@users.noreply.github.com>
veeti <veeti.paananen@rojekti.fi>
Vilbrekin <vilbrekin@gmail.com>
wkennington <william@wkennington.com>
WSGCSysadmin <e.meitner@willystreet.coop>
wweich <wweich@users.noreply.github.com>
wweich <wweich@gmx.de>
wweich <wulf@weich-kr.de>
xduugu <cedric@gmx.ca>
zaynetro <romanznet@gmail.com>
Zillode <zillode@zillode.be>
zukoo <fxgsell@gmail.com>

32
PULL_REQUEST_TEMPLATE.md Normal file
View File

@@ -0,0 +1,32 @@
### Purpose
Describe the purpose of this change. If there is an existing issue that is
resolved by this pull request, ensure that the commit subject is on the form
`Some short description (fixes #1234)` where 1234 is the issue number.
### Testing
Describe what testing has been done, and how the reviewer can test the change
if new tests are not included.
### Screenshots
If this is a GUI change, include screenshots of the change. If not, please
feel free to just delete this section.
### Documentation
If this is a user visible change (including API and protocol changes), add a link here
to the corresponding pull request on https://github.com/syncthing/docs or describe
the documentation changes necessary.
### Authorship
Every author of a code contribution (Go, Javascript, HTML, CSS etc, with the
possible exception of minor typo corrections and similar) is recorded in the
AUTHORS and NICKS files and the in-GUI credits. If this is your first
contribution, a maintainer will add you properly before accepting the
contribution. You need not do so yourself or worry about the fact that the
"authors" automated test fails. However, if your name (such as you want it
presented in the credits) is not visible on your Github profile or in your
commit messages, please assist by providing it here.

View File

@@ -1,112 +0,0 @@
# Docker Container for Syncthing
Use the Dockerfile in this repo, or pull the `syncthing/syncthing` image
from Docker Hub.
Use the `/var/syncthing` volume to have the synchronized files available on the
host. You can add more folders and map them as you prefer.
Note that Syncthing runs as UID 1000 and GID 1000 by default. These may be
altered with the `PUID` and `PGID` environment variables. In addition
the name of the Syncthing instance can be optionally defined by using
`--hostname=syncthing` parameter.
To grant Syncthing additional capabilities without running as root, use the
`PCAP` environment variable with the same syntax as that for `setcap(8)`.
For example, `PCAP=cap_chown,cap_fowner+ep`.
To set a different umask value, use the `UMASK` environment variable. For
example `UMASK=002`.
## Example Usage
**Docker cli**
```
$ docker pull syncthing/syncthing
$ docker run -p 8384:8384 -p 22000:22000/tcp -p 22000:22000/udp -p 21027:21027/udp \
-v /wherever/st-sync:/var/syncthing \
--hostname=my-syncthing \
syncthing/syncthing:latest
```
**Docker compose**
```yml
---
version: "3"
services:
syncthing:
image: syncthing/syncthing
container_name: syncthing
hostname: my-syncthing
environment:
- PUID=1000
- PGID=1000
volumes:
- /wherever/st-sync:/var/syncthing
ports:
- 8384:8384 # Web UI
- 22000:22000/tcp # TCP file transfers
- 22000:22000/udp # QUIC file transfers
- 21027:21027/udp # Receive local discovery broadcasts
restart: unless-stopped
```
## Discovery
Note that Docker's default network mode prevents local IP addresses from
being discovered, as Syncthing is only able to see the internal IP of the
container on the `172.17.0.0/16` subnet. This will result in poor transfer rates
if local device addresses are not manually configured.
It is therefore advisable to use the [host network mode](https://docs.docker.com/network/host/) instead:
**Docker cli**
```
$ docker pull syncthing/syncthing
$ docker run --network=host \
-v /wherever/st-sync:/var/syncthing \
syncthing/syncthing:latest
```
**Docker compose**
```yml
---
version: "3"
services:
syncthing:
image: syncthing/syncthing
container_name: syncthing
hostname: my-syncthing
environment:
- PUID=1000
- PGID=1000
volumes:
- /wherever/st-sync:/var/syncthing
network_mode: host
restart: unless-stopped
```
Be aware that syncthing alone is now in control of what interfaces and ports it
listens on. You can edit the syncthing configuration to change the defaults if
there are conflicts.
## GUI Security
By default Syncthing inside the Docker image listens on 0.0.0.0:8384 to
allow GUI connections via the Docker proxy. This is set by the
`STGUIADDRESS` environment variable in the Dockerfile, as it differs from
what Syncthing would otherwise use by default. This means you should set up
authentication in the GUI, like for any other externally reachable Syncthing
instance. If you do not require the GUI, or you use host networking, you can
unset the `STGUIADDRESS` variable to have Syncthing fall back to listening
on 127.0.0.1:
```
$ docker pull syncthing/syncthing
$ docker run -e STGUIADDRESS= \
-v /wherever/st-sync:/var/syncthing \
syncthing/syncthing:latest
```
With the environment variable unset Syncthing will follow what is set in the
configuration file / GUI settings dialog.

View File

@@ -2,6 +2,11 @@
---
[![Latest Linux & Cross Build](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square&label=linux+%26+cross)](https://build.syncthing.net/job/syncthing/lastBuild/)
[![Latest Windows Build](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing-windows.svg?style=flat-square&label=windows)](https://build.syncthing.net/job/syncthing/lastBuild/)
[![Latest Mac Build](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing-mac.svg?style=flat-square&label=mac)](https://build.syncthing.net/job/syncthing/lastBuild/)
[![Latest Solaris Build](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing-solaris.svg?style=flat-square&label=solaris)](https://build.syncthing.net/job/syncthing/lastBuild/)
[![API Documentation](https://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](https://godoc.org/github.com/syncthing/syncthing)
[![MPLv2 License](https://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/88/badge)](https://bestpractices.coreinfrastructure.org/projects/88)
[![Go Report Card](https://goreportcard.com/badge/github.com/syncthing/syncthing)](https://goreportcard.com/report/github.com/syncthing/syncthing)
@@ -10,42 +15,42 @@
Syncthing is a **continuous file synchronization program**. It synchronizes
files between two or more computers. We strive to fulfill the goals below.
The goals are listed in order of importance, the most important ones first.
This is the summary version of the goal list - for more
The goals are listed in order of importance, the most important one being
the first. This is the summary version of the goal list - for more
commentary, see the full [Goals document][13].
Syncthing should be:
1. **Safe From Data Loss**
1. Safe From Data Loss
Protecting the user's data is paramount. We take every reasonable
precaution to avoid corrupting the user's files.
2. **Secure Against Attackers**
2. Secure Against Attackers
Again, protecting the user's data is paramount. Regardless of our other
goals, we must never allow the user's data to be susceptible to
goals we must never allow the user's data to be susceptible to
eavesdropping or modification by unauthorized parties.
3. **Easy to Use**
3. Easy to Use
Syncthing should be approachable, understandable, and inclusive.
Syncthing should be approachable, understandable and inclusive.
4. **Automatic**
4. Automatic
User interaction should be required only when absolutely necessary.
5. **Universally Available**
5. Universally Available
Syncthing should run on every common computer. We are mindful that the
latest technology is not always available to every individual.
latest technology is not always available to any given individual.
6. **For Individuals**
6. For Individuals
Syncthing is primarily about empowering the individual user with safe,
secure, and easy to use file synchronization.
secure and easy to use file synchronization.
7. **Everything Else**
7. Everything Else
There are many things we care about that don't make it on to the list. It
is fine to optimize for these values, as long as they are not in conflict
@@ -57,56 +62,53 @@ Take a look at the [getting started guide][2].
There are a few examples for keeping Syncthing running in the background
on your system in [the etc directory][3]. There are also several [GUI
implementations][11] for Windows, Mac, and Linux.
implementations][11] for Windows, Mac and Linux.
## Docker
## Vote on features/bugs
To run Syncthing in Docker, see [the Docker README][16].
We'd like to encourage you to [vote][12] on issues that matter to you.
This helps the team understand what are the biggest pain points for our users, and could potentially influence what is being worked on next.
## Getting in Touch
The first and best point of contact is the [Forum][8].
If you've found something that is clearly a
The first and best point of contact is the [Forum][8]. There is also an IRC
channel, `#syncthing` on [freenode][4] (with a [web client][9]), for talking
directly to developers and users. If you've found something that is clearly a
bug, feel free to report it in the [GitHub issue tracker][10].
If you believe that youve found a Syncthing-related security vulnerability,
please report it by emailing security@syncthing.net. Do not report it in the
Forum or issue tracker.
## Building
Building Syncthing from source is easy. After extracting the source bundle from
a release or checking out git, you just need to run `go run build.go` and the
binaries are created in `./bin`. There's [a guide][5] with more details on the
build process.
Building Syncthing from source is easy, and there's a [guide][5]
that describes it for both Unix and Windows systems.
## Signed Releases
As of v0.10.15 and onwards, release binaries are GPG signed with the key
D26E6ED000654A3E, available from https://syncthing.net/security/ and
As of v0.10.15 and onwards release binaries are GPG signed with the key
D26E6ED000654A3E, available from https://syncthing.net/security.html and
most key servers.
There is also a built-in automatic upgrade mechanism (disabled in some
distribution channels) which uses a compiled in ECDSA signature. macOS
binaries are also properly code signed.
There is also a built in automatic upgrade mechanism (disabled in some
distribution channels) which uses a compiled in ECDSA signature. Mac OS
X binaries are also properly code signed.
## Documentation
Please see the Syncthing [documentation site][6] [[source]][17].
Please see the [Syncthing documentation site][6].
All code is licensed under the [MPLv2 License][7].
[1]: https://docs.syncthing.net/specs/bep-v1.html
[2]: https://docs.syncthing.net/intro/getting-started.html
[3]: https://github.com/syncthing/syncthing/blob/main/etc
[3]: https://github.com/syncthing/syncthing/blob/master/etc
[4]: https://www.freenode.net/
[5]: https://docs.syncthing.net/dev/building.html
[6]: https://docs.syncthing.net/
[7]: https://github.com/syncthing/syncthing/blob/main/LICENSE
[7]: https://github.com/syncthing/syncthing/blob/master/LICENSE
[8]: https://forum.syncthing.net/
[9]: https://kiwiirc.com/client/irc.freenode.net/#syncthing
[10]: https://github.com/syncthing/syncthing/issues
[11]: https://docs.syncthing.net/users/contrib.html#gui-wrappers
[13]: https://github.com/syncthing/syncthing/blob/main/GOALS.md
[12]: https://www.bountysource.com/teams/syncthing/issues
[13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
[14]: assets/logo-text-128.png
[15]: https://syncthing.net/
[16]: https://github.com/syncthing/syncthing/blob/main/README-Docker.md
[17]: https://github.com/syncthing/docs
[15]: https://syncthing.net/

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.5 KiB

After

Width:  |  Height:  |  Size: 9.8 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 20 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.2 KiB

After

Width:  |  Height:  |  Size: 2.2 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 38 KiB

After

Width:  |  Height:  |  Size: 40 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.8 KiB

After

Width:  |  Height:  |  Size: 4.9 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 19 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 36 KiB

After

Width:  |  Height:  |  Size: 38 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.5 KiB

After

Width:  |  Height:  |  Size: 9.8 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.1 KiB

After

Width:  |  Height:  |  Size: 8.2 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 160 KiB

View File

Binary file not shown.

1061
build.go
View File

File diff suppressed because it is too large Load Diff

View File

@@ -1,20 +0,0 @@
function build {
go run build.go @args
}
$cmd, $rest = $args
switch ($cmd) {
"test" {
$env:LOGGER_DISCARD=1
build test
}
"bench" {
$env:LOGGER_DISCARD=1
build bench
}
default {
build @rest
}
}

View File

@@ -2,6 +2,8 @@
set -euo pipefail
IFS=$'\n\t'
STTRACE=${STTRACE:-}
script() {
name="$1"
shift
@@ -13,23 +15,94 @@ build() {
}
case "${1:-default}" in
default)
build
;;
clean)
build "$@"
;;
tar)
build "$@"
;;
assets)
build "$@"
;;
xdr)
build "$@"
;;
translate)
build "$@"
;;
deb)
build "$@"
;;
setup)
build "$@"
;;
test)
ulimit -t 600 &>/dev/null || true
ulimit -d 512000 &>/dev/null || true
ulimit -m 512000 &>/dev/null || true
LOGGER_DISCARD=1 build test
;;
bench)
LOGGER_DISCARD=1 build bench
LOGGER_DISCARD=1 build bench | script benchfilter
;;
prerelease)
script authors
build weblate
go run script/authors.go
build transifex
pushd man ; ./refresh.sh ; popd
git add -A gui man AUTHORS
git commit -m 'gui, man, authors: Update docs, translations, and contributors'
git add -A gui man
git commit -m 'gui, man: Update docs & translations'
;;
noupgrade)
build -no-upgrade tar
;;
all)
platforms=(
darwin-amd64 dragonfly-amd64 freebsd-amd64 linux-amd64 netbsd-amd64 openbsd-amd64 solaris-amd64 windows-amd64
freebsd-386 linux-386 netbsd-386 openbsd-386 windows-386
linux-arm linux-arm64 linux-ppc64 linux-ppc64le
)
for plat in "${platforms[@]}"; do
echo Building "$plat"
goos="${plat%-*}"
goarch="${plat#*-}"
dist="tar"
if [[ $goos == "windows" ]]; then
dist="zip"
fi
build -goos "$goos" -goarch "$goarch" "$dist"
echo
done
;;
test-xunit)
ulimit -t 600 &>/dev/null || true
ulimit -d 512000 &>/dev/null || true
ulimit -m 512000 &>/dev/null || true
(GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -v -race ./lib/... ./cmd/... || true) > tests.out
go2xunit -output tests.xml -fail < tests.out
;;
*)
build "$@"
echo "Unknown build command $1"
;;
esac

143
cmd/stbench/main.go Normal file
View File

@@ -0,0 +1,143 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// This doesn't build on Windows due to the Rusage stuff.
// +build !windows
package main
import (
"flag"
"fmt"
"log"
"runtime"
"syscall"
"time"
"github.com/syncthing/syncthing/lib/rc"
)
var homeDir = "h1"
var syncthingBin = "./bin/syncthing"
var test = "scan"
func main() {
flag.StringVar(&homeDir, "home", homeDir, "Home directory location")
flag.StringVar(&syncthingBin, "bin", syncthingBin, "Binary location")
flag.StringVar(&test, "test", test, "Test to run")
flag.Parse()
switch test {
case "scan":
// scan measures the resource usage required to perform the initial
// scan, without cleaning away the database first.
testScan()
}
}
// testScan starts a process and reports on the resource usage required to
// perform the initial scan.
func testScan() {
log.Println("Starting...")
p := rc.NewProcess("127.0.0.1:8081")
if err := p.Start(syncthingBin, "-home", homeDir, "-no-browser"); err != nil {
log.Println(err)
return
}
defer p.Stop()
wallTime := awaitScanComplete(p)
report(p, wallTime)
}
// awaitScanComplete waits for a folder to transition idle->scanning and
// then scanning->idle and returns the time taken for the scan.
func awaitScanComplete(p *rc.Process) time.Duration {
log.Println("Awaiting scan completion...")
var t0, t1 time.Time
lastEvent := 0
loop:
for {
evs, err := p.Events(lastEvent)
if err != nil {
continue
}
for _, ev := range evs {
if ev.Type == "StateChanged" {
data := ev.Data.(map[string]interface{})
log.Println(ev)
if data["to"].(string) == "scanning" {
t0 = ev.Time
continue
}
if !t0.IsZero() && data["to"].(string) == "idle" {
t1 = ev.Time
break loop
}
}
lastEvent = ev.ID
}
time.Sleep(250 * time.Millisecond)
}
return t1.Sub(t0)
}
// report stops the given process and reports on it's resource usage in two
// ways: human readable to stderr, and CSV to stdout.
func report(p *rc.Process, wallTime time.Duration) {
sv, err := p.SystemVersion()
if err != nil {
log.Println(err)
return
}
ss, err := p.SystemStatus()
if err != nil {
log.Println(err)
return
}
proc, err := p.Stop()
if err != nil {
return
}
rusage, ok := proc.SysUsage().(*syscall.Rusage)
if !ok {
return
}
log.Println("Version:", sv.Version)
log.Println("Alloc:", ss.Alloc/1024, "KiB")
log.Println("Sys:", ss.Sys/1024, "KiB")
log.Println("Goroutines:", ss.Goroutines)
log.Println("Wall time:", wallTime)
log.Println("Utime:", time.Duration(rusage.Utime.Nano()))
log.Println("Stime:", time.Duration(rusage.Stime.Nano()))
if runtime.GOOS == "darwin" {
// Darwin reports in bytes, Linux seems to report in KiB even
// though the manpage says otherwise.
rusage.Maxrss /= 1024
}
log.Println("MaxRSS:", rusage.Maxrss, "KiB")
fmt.Printf("%s,%d,%d,%d,%.02f,%.02f,%.02f,%d\n",
sv.Version,
ss.Alloc/1024,
ss.Sys/1024,
ss.Goroutines,
wallTime.Seconds(),
time.Duration(rusage.Utime.Nano()).Seconds(),
time.Duration(rusage.Stime.Nano()).Seconds(),
rusage.Maxrss)
}

19
cmd/stcli/LICENSE Normal file
View File

@@ -0,0 +1,19 @@
Copyright (C) 2014 Audrius Butkevičius
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

115
cmd/stcli/client.go Normal file
View File

@@ -0,0 +1,115 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"bytes"
"crypto/tls"
"net/http"
"strings"
"github.com/AudriusButkevicius/cli"
)
type APIClient struct {
httpClient http.Client
endpoint string
apikey string
username string
password string
id string
csrf string
}
var instance *APIClient
func getClient(c *cli.Context) *APIClient {
if instance != nil {
return instance
}
endpoint := c.GlobalString("endpoint")
if !strings.HasPrefix(endpoint, "http") {
endpoint = "http://" + endpoint
}
httpClient := http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: c.GlobalBool("insecure"),
},
},
}
client := APIClient{
httpClient: httpClient,
endpoint: endpoint,
apikey: c.GlobalString("apikey"),
username: c.GlobalString("username"),
password: c.GlobalString("password"),
}
if client.apikey == "" {
request, err := http.NewRequest("GET", client.endpoint, nil)
die(err)
response := client.handleRequest(request)
client.id = response.Header.Get("X-Syncthing-ID")
if client.id == "" {
die("Failed to get device ID")
}
for _, item := range response.Cookies() {
if item.Name == "CSRF-Token-"+client.id[:5] {
client.csrf = item.Value
goto csrffound
}
}
die("Failed to get CSRF token")
csrffound:
}
instance = &client
return &client
}
func (client *APIClient) handleRequest(request *http.Request) *http.Response {
if client.apikey != "" {
request.Header.Set("X-API-Key", client.apikey)
}
if client.username != "" || client.password != "" {
request.SetBasicAuth(client.username, client.password)
}
if client.csrf != "" {
request.Header.Set("X-CSRF-Token-"+client.id[:5], client.csrf)
}
response, err := client.httpClient.Do(request)
die(err)
if response.StatusCode == 404 {
die("Invalid endpoint or API call")
} else if response.StatusCode == 401 {
die("Invalid username or password")
} else if response.StatusCode == 403 {
if client.apikey == "" {
die("Invalid CSRF token")
}
die("Invalid API key")
} else if response.StatusCode != 200 {
body := strings.TrimSpace(string(responseToBArray(response)))
if body != "" {
die(body)
}
die("Unknown HTTP status returned: " + response.Status)
}
return response
}
func httpGet(c *cli.Context, url string) *http.Response {
client := getClient(c)
request, err := http.NewRequest("GET", client.endpoint+"/rest/"+url, nil)
die(err)
return client.handleRequest(request)
}
func httpPost(c *cli.Context, url string, body string) *http.Response {
client := getClient(c)
request, err := http.NewRequest("POST", client.endpoint+"/rest/"+url, bytes.NewBufferString(body))
die(err)
return client.handleRequest(request)
}

188
cmd/stcli/cmd_devices.go Normal file
View File

@@ -0,0 +1,188 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"fmt"
"strings"
"github.com/AudriusButkevicius/cli"
"github.com/syncthing/syncthing/lib/config"
)
func init() {
cliCommands = append(cliCommands, cli.Command{
Name: "devices",
HideHelp: true,
Usage: "Device command group",
Subcommands: []cli.Command{
{
Name: "list",
Usage: "List registered devices",
Requires: &cli.Requires{},
Action: devicesList,
},
{
Name: "add",
Usage: "Add a new device",
Requires: &cli.Requires{"device id", "device name?"},
Action: devicesAdd,
},
{
Name: "remove",
Usage: "Remove an existing device",
Requires: &cli.Requires{"device id"},
Action: devicesRemove,
},
{
Name: "get",
Usage: "Get a property of a device",
Requires: &cli.Requires{"device id", "property"},
Action: devicesGet,
},
{
Name: "set",
Usage: "Set a property of a device",
Requires: &cli.Requires{"device id", "property", "value..."},
Action: devicesSet,
},
},
})
}
func devicesList(c *cli.Context) {
cfg := getConfig(c)
first := true
writer := newTableWriter()
for _, device := range cfg.Devices {
if !first {
fmt.Fprintln(writer)
}
fmt.Fprintln(writer, "ID:\t", device.DeviceID, "\t")
fmt.Fprintln(writer, "Name:\t", device.Name, "\t(name)")
fmt.Fprintln(writer, "Address:\t", strings.Join(device.Addresses, " "), "\t(address)")
fmt.Fprintln(writer, "Compression:\t", device.Compression, "\t(compression)")
fmt.Fprintln(writer, "Certificate name:\t", device.CertName, "\t(certname)")
fmt.Fprintln(writer, "Introducer:\t", device.Introducer, "\t(introducer)")
first = false
}
writer.Flush()
}
func devicesAdd(c *cli.Context) {
nid := c.Args()[0]
id := parseDeviceID(nid)
newDevice := config.DeviceConfiguration{
DeviceID: id,
Name: nid,
Addresses: []string{"dynamic"},
}
if len(c.Args()) > 1 {
newDevice.Name = c.Args()[1]
}
if len(c.Args()) > 2 {
addresses := c.Args()[2:]
for _, item := range addresses {
if item == "dynamic" {
continue
}
validAddress(item)
}
newDevice.Addresses = addresses
}
cfg := getConfig(c)
for _, device := range cfg.Devices {
if device.DeviceID == id {
die("Device " + nid + " already exists")
}
}
cfg.Devices = append(cfg.Devices, newDevice)
setConfig(c, cfg)
}
func devicesRemove(c *cli.Context) {
nid := c.Args()[0]
id := parseDeviceID(nid)
if nid == getMyID(c) {
die("Cannot remove yourself")
}
cfg := getConfig(c)
for i, device := range cfg.Devices {
if device.DeviceID == id {
last := len(cfg.Devices) - 1
cfg.Devices[i] = cfg.Devices[last]
cfg.Devices = cfg.Devices[:last]
setConfig(c, cfg)
return
}
}
die("Device " + nid + " not found")
}
func devicesGet(c *cli.Context) {
nid := c.Args()[0]
id := parseDeviceID(nid)
arg := c.Args()[1]
cfg := getConfig(c)
for _, device := range cfg.Devices {
if device.DeviceID != id {
continue
}
switch strings.ToLower(arg) {
case "name":
fmt.Println(device.Name)
case "address":
fmt.Println(strings.Join(device.Addresses, "\n"))
case "compression":
fmt.Println(device.Compression.String())
case "certname":
fmt.Println(device.CertName)
case "introducer":
fmt.Println(device.Introducer)
default:
die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
}
return
}
die("Device " + nid + " not found")
}
func devicesSet(c *cli.Context) {
nid := c.Args()[0]
id := parseDeviceID(nid)
arg := c.Args()[1]
config := getConfig(c)
for i, device := range config.Devices {
if device.DeviceID != id {
continue
}
switch strings.ToLower(arg) {
case "name":
config.Devices[i].Name = strings.Join(c.Args()[2:], " ")
case "address":
for _, item := range c.Args()[2:] {
if item == "dynamic" {
continue
}
validAddress(item)
}
config.Devices[i].Addresses = c.Args()[2:]
case "compression":
err := config.Devices[i].Compression.UnmarshalText([]byte(c.Args()[2]))
die(err)
case "certname":
config.Devices[i].CertName = strings.Join(c.Args()[2:], " ")
case "introducer":
config.Devices[i].Introducer = parseBool(c.Args()[2])
default:
die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
}
setConfig(c, config)
return
}
die("Device " + nid + " not found")
}

67
cmd/stcli/cmd_errors.go Normal file
View File

@@ -0,0 +1,67 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"encoding/json"
"fmt"
"strings"
"github.com/AudriusButkevicius/cli"
)
func init() {
cliCommands = append(cliCommands, cli.Command{
Name: "errors",
HideHelp: true,
Usage: "Error command group",
Subcommands: []cli.Command{
{
Name: "show",
Usage: "Show pending errors",
Requires: &cli.Requires{},
Action: errorsShow,
},
{
Name: "push",
Usage: "Push an error to active clients",
Requires: &cli.Requires{"error message..."},
Action: errorsPush,
},
{
Name: "clear",
Usage: "Clear pending errors",
Requires: &cli.Requires{},
Action: wrappedHTTPPost("system/error/clear"),
},
},
})
}
func errorsShow(c *cli.Context) {
response := httpGet(c, "system/error")
var data map[string][]map[string]interface{}
json.Unmarshal(responseToBArray(response), &data)
writer := newTableWriter()
for _, item := range data["errors"] {
time := item["time"].(string)[:19]
time = strings.Replace(time, "T", " ", 1)
err := item["error"].(string)
err = strings.TrimSpace(err)
fmt.Fprintln(writer, time+":\t"+err)
}
writer.Flush()
}
func errorsPush(c *cli.Context) {
err := strings.Join(c.Args(), " ")
response := httpPost(c, "system/error", strings.TrimSpace(err))
if response.StatusCode != 200 {
err = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
body := string(responseToBArray(response))
if body != "" {
err += "\nBody: " + body
}
die(err)
}
}

351
cmd/stcli/cmd_folders.go Normal file
View File

@@ -0,0 +1,351 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"fmt"
"path/filepath"
"strings"
"github.com/AudriusButkevicius/cli"
"github.com/syncthing/syncthing/lib/config"
)
func init() {
cliCommands = append(cliCommands, cli.Command{
Name: "folders",
HideHelp: true,
Usage: "Folder command group",
Subcommands: []cli.Command{
{
Name: "list",
Usage: "List available folders",
Requires: &cli.Requires{},
Action: foldersList,
},
{
Name: "add",
Usage: "Add a new folder",
Requires: &cli.Requires{"folder id", "directory"},
Action: foldersAdd,
},
{
Name: "remove",
Usage: "Remove an existing folder",
Requires: &cli.Requires{"folder id"},
Action: foldersRemove,
},
{
Name: "override",
Usage: "Override changes from other nodes for a master folder",
Requires: &cli.Requires{"folder id"},
Action: foldersOverride,
},
{
Name: "get",
Usage: "Get a property of a folder",
Requires: &cli.Requires{"folder id", "property"},
Action: foldersGet,
},
{
Name: "set",
Usage: "Set a property of a folder",
Requires: &cli.Requires{"folder id", "property", "value..."},
Action: foldersSet,
},
{
Name: "unset",
Usage: "Unset a property of a folder",
Requires: &cli.Requires{"folder id", "property"},
Action: foldersUnset,
},
{
Name: "devices",
Usage: "Folder devices command group",
HideHelp: true,
Subcommands: []cli.Command{
{
Name: "list",
Usage: "List of devices which the folder is shared with",
Requires: &cli.Requires{"folder id"},
Action: foldersDevicesList,
},
{
Name: "add",
Usage: "Share a folder with a device",
Requires: &cli.Requires{"folder id", "device id"},
Action: foldersDevicesAdd,
},
{
Name: "remove",
Usage: "Unshare a folder with a device",
Requires: &cli.Requires{"folder id", "device id"},
Action: foldersDevicesRemove,
},
{
Name: "clear",
Usage: "Unshare a folder with all devices",
Requires: &cli.Requires{"folder id"},
Action: foldersDevicesClear,
},
},
},
},
})
}
func foldersList(c *cli.Context) {
cfg := getConfig(c)
first := true
writer := newTableWriter()
for _, folder := range cfg.Folders {
if !first {
fmt.Fprintln(writer)
}
fmt.Fprintln(writer, "ID:\t", folder.ID, "\t")
fmt.Fprintln(writer, "Path:\t", folder.RawPath, "\t(directory)")
fmt.Fprintln(writer, "Folder type:\t", folder.Type, "\t(type)")
fmt.Fprintln(writer, "Ignore permissions:\t", folder.IgnorePerms, "\t(permissions)")
fmt.Fprintln(writer, "Rescan interval in seconds:\t", folder.RescanIntervalS, "\t(rescan)")
if folder.Versioning.Type != "" {
fmt.Fprintln(writer, "Versioning:\t", folder.Versioning.Type, "\t(versioning)")
for key, value := range folder.Versioning.Params {
fmt.Fprintf(writer, "Versioning %s:\t %s \t(versioning-%s)\n", key, value, key)
}
}
first = false
}
writer.Flush()
}
func foldersAdd(c *cli.Context) {
cfg := getConfig(c)
abs, err := filepath.Abs(c.Args()[1])
die(err)
folder := config.FolderConfiguration{
ID: c.Args()[0],
RawPath: filepath.Clean(abs),
}
cfg.Folders = append(cfg.Folders, folder)
setConfig(c, cfg)
}
func foldersRemove(c *cli.Context) {
cfg := getConfig(c)
rid := c.Args()[0]
for i, folder := range cfg.Folders {
if folder.ID == rid {
last := len(cfg.Folders) - 1
cfg.Folders[i] = cfg.Folders[last]
cfg.Folders = cfg.Folders[:last]
setConfig(c, cfg)
return
}
}
die("Folder " + rid + " not found")
}
func foldersOverride(c *cli.Context) {
cfg := getConfig(c)
rid := c.Args()[0]
for _, folder := range cfg.Folders {
if folder.ID == rid && folder.Type == config.FolderTypeSendOnly {
response := httpPost(c, "db/override", "")
if response.StatusCode != 200 {
err := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
body := string(responseToBArray(response))
if body != "" {
err += "\nBody: " + body
}
die(err)
}
return
}
}
die("Folder " + rid + " not found or folder not master")
}
func foldersGet(c *cli.Context) {
cfg := getConfig(c)
rid := c.Args()[0]
arg := strings.ToLower(c.Args()[1])
for _, folder := range cfg.Folders {
if folder.ID != rid {
continue
}
if strings.HasPrefix(arg, "versioning-") {
arg = arg[11:]
value, ok := folder.Versioning.Params[arg]
if ok {
fmt.Println(value)
return
}
die("Versioning property " + c.Args()[1][11:] + " not found")
}
switch arg {
case "directory":
fmt.Println(folder.RawPath)
case "type":
fmt.Println(folder.Type)
case "permissions":
fmt.Println(folder.IgnorePerms)
case "rescan":
fmt.Println(folder.RescanIntervalS)
case "versioning":
if folder.Versioning.Type != "" {
fmt.Println(folder.Versioning.Type)
}
default:
die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, type, permissions, versioning, versioning-<key>")
}
return
}
die("Folder " + rid + " not found")
}
func foldersSet(c *cli.Context) {
rid := c.Args()[0]
arg := strings.ToLower(c.Args()[1])
val := strings.Join(c.Args()[2:], " ")
cfg := getConfig(c)
for i, folder := range cfg.Folders {
if folder.ID != rid {
continue
}
if strings.HasPrefix(arg, "versioning-") {
cfg.Folders[i].Versioning.Params[arg[11:]] = val
setConfig(c, cfg)
return
}
switch arg {
case "directory":
cfg.Folders[i].RawPath = val
case "type":
var t config.FolderType
if err := t.UnmarshalText([]byte(val)); err != nil {
die("Invalid folder type: " + err.Error())
}
cfg.Folders[i].Type = t
case "permissions":
cfg.Folders[i].IgnorePerms = parseBool(val)
case "rescan":
cfg.Folders[i].RescanIntervalS = parseInt(val)
case "versioning":
cfg.Folders[i].Versioning.Type = val
default:
die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, master, permissions, versioning, versioning-<key>")
}
setConfig(c, cfg)
return
}
die("Folder " + rid + " not found")
}
func foldersUnset(c *cli.Context) {
rid := c.Args()[0]
arg := strings.ToLower(c.Args()[1])
cfg := getConfig(c)
for i, folder := range cfg.Folders {
if folder.ID != rid {
continue
}
if strings.HasPrefix(arg, "versioning-") {
arg = arg[11:]
if _, ok := folder.Versioning.Params[arg]; ok {
delete(cfg.Folders[i].Versioning.Params, arg)
setConfig(c, cfg)
return
}
die("Versioning property " + c.Args()[1][11:] + " not found")
}
switch arg {
case "versioning":
cfg.Folders[i].Versioning.Type = ""
cfg.Folders[i].Versioning.Params = make(map[string]string)
default:
die("Invalid property: " + c.Args()[1] + "\nAvailable properties: versioning, versioning-<key>")
}
setConfig(c, cfg)
return
}
die("Folder " + rid + " not found")
}
func foldersDevicesList(c *cli.Context) {
rid := c.Args()[0]
cfg := getConfig(c)
for _, folder := range cfg.Folders {
if folder.ID != rid {
continue
}
for _, device := range folder.Devices {
fmt.Println(device.DeviceID)
}
return
}
die("Folder " + rid + " not found")
}
func foldersDevicesAdd(c *cli.Context) {
rid := c.Args()[0]
nid := parseDeviceID(c.Args()[1])
cfg := getConfig(c)
for i, folder := range cfg.Folders {
if folder.ID != rid {
continue
}
for _, device := range folder.Devices {
if device.DeviceID == nid {
die("Device " + c.Args()[1] + " is already part of this folder")
}
}
for _, device := range cfg.Devices {
if device.DeviceID == nid {
cfg.Folders[i].Devices = append(folder.Devices, config.FolderDeviceConfiguration{
DeviceID: device.DeviceID,
})
setConfig(c, cfg)
return
}
}
die("Device " + c.Args()[1] + " not found in device list")
}
die("Folder " + rid + " not found")
}
func foldersDevicesRemove(c *cli.Context) {
rid := c.Args()[0]
nid := parseDeviceID(c.Args()[1])
cfg := getConfig(c)
for ri, folder := range cfg.Folders {
if folder.ID != rid {
continue
}
for ni, device := range folder.Devices {
if device.DeviceID == nid {
last := len(folder.Devices) - 1
cfg.Folders[ri].Devices[ni] = folder.Devices[last]
cfg.Folders[ri].Devices = cfg.Folders[ri].Devices[:last]
setConfig(c, cfg)
return
}
}
die("Device " + c.Args()[1] + " not found")
}
die("Folder " + rid + " not found")
}
func foldersDevicesClear(c *cli.Context) {
rid := c.Args()[0]
cfg := getConfig(c)
for i, folder := range cfg.Folders {
if folder.ID != rid {
continue
}
cfg.Folders[i].Devices = []config.FolderDeviceConfiguration{}
setConfig(c, cfg)
return
}
die("Folder " + rid + " not found")
}

78
cmd/stcli/cmd_general.go Normal file
View File

@@ -0,0 +1,78 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"encoding/json"
"fmt"
"github.com/AudriusButkevicius/cli"
)
func init() {
cliCommands = append(cliCommands, []cli.Command{
{
Name: "id",
Usage: "Get ID of the Syncthing client",
Requires: &cli.Requires{},
Action: generalID,
},
{
Name: "status",
Usage: "Configuration status, whether or not a restart is required for changes to take effect",
Requires: &cli.Requires{},
Action: generalStatus,
},
{
Name: "restart",
Usage: "Restart syncthing",
Requires: &cli.Requires{},
Action: wrappedHTTPPost("system/restart"),
},
{
Name: "shutdown",
Usage: "Shutdown syncthing",
Requires: &cli.Requires{},
Action: wrappedHTTPPost("system/shutdown"),
},
{
Name: "reset",
Usage: "Reset syncthing deleting all folders and devices",
Requires: &cli.Requires{},
Action: wrappedHTTPPost("system/reset"),
},
{
Name: "upgrade",
Usage: "Upgrade syncthing (if a newer version is available)",
Requires: &cli.Requires{},
Action: wrappedHTTPPost("system/upgrade"),
},
{
Name: "version",
Usage: "Syncthing client version",
Requires: &cli.Requires{},
Action: generalVersion,
},
}...)
}
func generalID(c *cli.Context) {
fmt.Println(getMyID(c))
}
func generalStatus(c *cli.Context) {
response := httpGet(c, "system/config/insync")
var status struct{ ConfigInSync bool }
json.Unmarshal(responseToBArray(response), &status)
if !status.ConfigInSync {
die("Config out of sync")
}
fmt.Println("Config in sync")
}
func generalVersion(c *cli.Context) {
response := httpGet(c, "system/version")
version := make(map[string]interface{})
json.Unmarshal(responseToBArray(response), &version)
prettyPrintJSON(version)
}

127
cmd/stcli/cmd_gui.go Normal file
View File

@@ -0,0 +1,127 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"fmt"
"strings"
"github.com/AudriusButkevicius/cli"
)
func init() {
cliCommands = append(cliCommands, cli.Command{
Name: "gui",
HideHelp: true,
Usage: "GUI command group",
Subcommands: []cli.Command{
{
Name: "dump",
Usage: "Show all GUI configuration settings",
Requires: &cli.Requires{},
Action: guiDump,
},
{
Name: "get",
Usage: "Get a GUI configuration setting",
Requires: &cli.Requires{"setting"},
Action: guiGet,
},
{
Name: "set",
Usage: "Set a GUI configuration setting",
Requires: &cli.Requires{"setting", "value"},
Action: guiSet,
},
{
Name: "unset",
Usage: "Unset a GUI configuration setting",
Requires: &cli.Requires{"setting"},
Action: guiUnset,
},
},
})
}
func guiDump(c *cli.Context) {
cfg := getConfig(c).GUI
writer := newTableWriter()
fmt.Fprintln(writer, "Enabled:\t", cfg.Enabled, "\t(enabled)")
fmt.Fprintln(writer, "Use HTTPS:\t", cfg.UseTLS(), "\t(tls)")
fmt.Fprintln(writer, "Listen Addresses:\t", cfg.Address(), "\t(address)")
if cfg.User != "" {
fmt.Fprintln(writer, "Authentication User:\t", cfg.User, "\t(username)")
fmt.Fprintln(writer, "Authentication Password:\t", cfg.Password, "\t(password)")
}
if cfg.APIKey != "" {
fmt.Fprintln(writer, "API Key:\t", cfg.APIKey, "\t(apikey)")
}
writer.Flush()
}
func guiGet(c *cli.Context) {
cfg := getConfig(c).GUI
arg := c.Args()[0]
switch strings.ToLower(arg) {
case "enabled":
fmt.Println(cfg.Enabled)
case "tls":
fmt.Println(cfg.UseTLS())
case "address":
fmt.Println(cfg.Address())
case "user":
if cfg.User != "" {
fmt.Println(cfg.User)
}
case "password":
if cfg.User != "" {
fmt.Println(cfg.Password)
}
case "apikey":
if cfg.APIKey != "" {
fmt.Println(cfg.APIKey)
}
default:
die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
}
}
func guiSet(c *cli.Context) {
cfg := getConfig(c)
arg := c.Args()[0]
val := c.Args()[1]
switch strings.ToLower(arg) {
case "enabled":
cfg.GUI.Enabled = parseBool(val)
case "tls":
cfg.GUI.RawUseTLS = parseBool(val)
case "address":
validAddress(val)
cfg.GUI.RawAddress = val
case "user":
cfg.GUI.User = val
case "password":
cfg.GUI.Password = val
case "apikey":
cfg.GUI.APIKey = val
default:
die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
}
setConfig(c, cfg)
}
func guiUnset(c *cli.Context) {
cfg := getConfig(c)
arg := c.Args()[0]
switch strings.ToLower(arg) {
case "user":
cfg.GUI.User = ""
case "password":
cfg.GUI.Password = ""
case "apikey":
cfg.GUI.APIKey = ""
default:
die("Invalid setting: " + arg + "\nAvailable settings: user, password, apikey")
}
setConfig(c, cfg)
}

173
cmd/stcli/cmd_options.go Normal file
View File

@@ -0,0 +1,173 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"fmt"
"strings"
"github.com/AudriusButkevicius/cli"
)
func init() {
cliCommands = append(cliCommands, cli.Command{
Name: "options",
HideHelp: true,
Usage: "Options command group",
Subcommands: []cli.Command{
{
Name: "dump",
Usage: "Show all Syncthing option settings",
Requires: &cli.Requires{},
Action: optionsDump,
},
{
Name: "get",
Usage: "Get a Syncthing option setting",
Requires: &cli.Requires{"setting"},
Action: optionsGet,
},
{
Name: "set",
Usage: "Set a Syncthing option setting",
Requires: &cli.Requires{"setting", "value..."},
Action: optionsSet,
},
},
})
}
func optionsDump(c *cli.Context) {
cfg := getConfig(c).Options
writer := newTableWriter()
fmt.Fprintln(writer, "Sync protocol listen addresses:\t", strings.Join(cfg.ListenAddresses, " "), "\t(addresses)")
fmt.Fprintln(writer, "Global discovery enabled:\t", cfg.GlobalAnnEnabled, "\t(globalannenabled)")
fmt.Fprintln(writer, "Global discovery servers:\t", strings.Join(cfg.GlobalAnnServers, " "), "\t(globalannserver)")
fmt.Fprintln(writer, "Local discovery enabled:\t", cfg.LocalAnnEnabled, "\t(localannenabled)")
fmt.Fprintln(writer, "Local discovery port:\t", cfg.LocalAnnPort, "\t(localannport)")
fmt.Fprintln(writer, "Outgoing rate limit in KiB/s:\t", cfg.MaxSendKbps, "\t(maxsend)")
fmt.Fprintln(writer, "Incoming rate limit in KiB/s:\t", cfg.MaxRecvKbps, "\t(maxrecv)")
fmt.Fprintln(writer, "Reconnect interval in seconds:\t", cfg.ReconnectIntervalS, "\t(reconnect)")
fmt.Fprintln(writer, "Start browser:\t", cfg.StartBrowser, "\t(browser)")
fmt.Fprintln(writer, "Enable UPnP:\t", cfg.NATEnabled, "\t(nat)")
fmt.Fprintln(writer, "UPnP Lease in minutes:\t", cfg.NATLeaseM, "\t(natlease)")
fmt.Fprintln(writer, "UPnP Renewal period in minutes:\t", cfg.NATRenewalM, "\t(natrenew)")
fmt.Fprintln(writer, "Restart on Wake Up:\t", cfg.RestartOnWakeup, "\t(wake)")
reporting := "unrecognized value"
switch cfg.URAccepted {
case -1:
reporting = "false"
case 0:
reporting = "undecided/false"
case 1:
reporting = "true"
}
fmt.Fprintln(writer, "Anonymous usage reporting:\t", reporting, "\t(reporting)")
writer.Flush()
}
func optionsGet(c *cli.Context) {
cfg := getConfig(c).Options
arg := c.Args()[0]
switch strings.ToLower(arg) {
case "address":
fmt.Println(strings.Join(cfg.ListenAddresses, "\n"))
case "globalannenabled":
fmt.Println(cfg.GlobalAnnEnabled)
case "globalannservers":
fmt.Println(strings.Join(cfg.GlobalAnnServers, "\n"))
case "localannenabled":
fmt.Println(cfg.LocalAnnEnabled)
case "localannport":
fmt.Println(cfg.LocalAnnPort)
case "maxsend":
fmt.Println(cfg.MaxSendKbps)
case "maxrecv":
fmt.Println(cfg.MaxRecvKbps)
case "reconnect":
fmt.Println(cfg.ReconnectIntervalS)
case "browser":
fmt.Println(cfg.StartBrowser)
case "nat":
fmt.Println(cfg.NATEnabled)
case "natlease":
fmt.Println(cfg.NATLeaseM)
case "natrenew":
fmt.Println(cfg.NATRenewalM)
case "reporting":
switch cfg.URAccepted {
case -1:
fmt.Println("false")
case 0:
fmt.Println("undecided/false")
case 1:
fmt.Println("true")
default:
fmt.Println("unknown")
}
case "wake":
fmt.Println(cfg.RestartOnWakeup)
default:
die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
}
}
func optionsSet(c *cli.Context) {
config := getConfig(c)
arg := c.Args()[0]
val := c.Args()[1]
switch strings.ToLower(arg) {
case "address":
for _, item := range c.Args().Tail() {
validAddress(item)
}
config.Options.ListenAddresses = c.Args().Tail()
case "globalannenabled":
config.Options.GlobalAnnEnabled = parseBool(val)
case "globalannserver":
for _, item := range c.Args().Tail() {
validAddress(item)
}
config.Options.GlobalAnnServers = c.Args().Tail()
case "localannenabled":
config.Options.LocalAnnEnabled = parseBool(val)
case "localannport":
config.Options.LocalAnnPort = parsePort(val)
case "maxsend":
config.Options.MaxSendKbps = parseUint(val)
case "maxrecv":
config.Options.MaxRecvKbps = parseUint(val)
case "reconnect":
config.Options.ReconnectIntervalS = parseUint(val)
case "browser":
config.Options.StartBrowser = parseBool(val)
case "nat":
config.Options.NATEnabled = parseBool(val)
case "natlease":
config.Options.NATLeaseM = parseUint(val)
case "natrenew":
config.Options.NATRenewalM = parseUint(val)
case "reporting":
switch strings.ToLower(val) {
case "u", "undecided", "unset":
config.Options.URAccepted = 0
default:
boolvalue := parseBool(val)
if boolvalue {
config.Options.URAccepted = 1
} else {
config.Options.URAccepted = -1
}
}
case "wake":
config.Options.RestartOnWakeup = parseBool(val)
default:
die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
}
setConfig(c, config)
}

72
cmd/stcli/cmd_report.go Normal file
View File

@@ -0,0 +1,72 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"encoding/json"
"fmt"
"github.com/AudriusButkevicius/cli"
)
func init() {
cliCommands = append(cliCommands, cli.Command{
Name: "report",
HideHelp: true,
Usage: "Reporting command group",
Subcommands: []cli.Command{
{
Name: "system",
Usage: "Report system state",
Requires: &cli.Requires{},
Action: reportSystem,
},
{
Name: "connections",
Usage: "Report about connections to other devices",
Requires: &cli.Requires{},
Action: reportConnections,
},
{
Name: "usage",
Usage: "Usage report",
Requires: &cli.Requires{},
Action: reportUsage,
},
},
})
}
func reportSystem(c *cli.Context) {
response := httpGet(c, "system/status")
data := make(map[string]interface{})
json.Unmarshal(responseToBArray(response), &data)
prettyPrintJSON(data)
}
func reportConnections(c *cli.Context) {
response := httpGet(c, "system/connections")
data := make(map[string]map[string]interface{})
json.Unmarshal(responseToBArray(response), &data)
var overall map[string]interface{}
for key, value := range data {
if key == "total" {
overall = value
continue
}
value["Device ID"] = key
prettyPrintJSON(value)
fmt.Println()
}
if overall != nil {
fmt.Println("=== Overall statistics ===")
prettyPrintJSON(overall)
}
}
func reportUsage(c *cli.Context) {
response := httpGet(c, "svc/report")
report := make(map[string]interface{})
json.Unmarshal(responseToBArray(response), &report)
prettyPrintJSON(report)
}

31
cmd/stcli/labels.go Normal file
View File

@@ -0,0 +1,31 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
var jsonAttributeLabels = map[string]string{
"folderMaxMiB": "Largest folder size in MiB",
"folderMaxFiles": "Largest folder file count",
"longVersion": "Long version",
"totMiB": "Total size in MiB",
"totFiles": "Total files",
"uniqueID": "Unique ID",
"numFolders": "Folder count",
"numDevices": "Device count",
"memoryUsageMiB": "Memory usage in MiB",
"memorySize": "Total memory in MiB",
"sha256Perf": "SHA256 Benchmark",
"At": "Last contacted",
"Completion": "Percent complete",
"InBytesTotal": "Total bytes received",
"OutBytesTotal": "Total bytes sent",
"ClientVersion": "Client version",
"alloc": "Memory allocated in bytes",
"sys": "Memory using in bytes",
"cpuPercent": "CPU load in percent",
"extAnnounceOK": "External announcments working",
"goroutines": "Number of Go routines",
"myID": "Client ID",
"tilde": "Tilde expands to",
"arch": "Architecture",
"os": "OS",
}

63
cmd/stcli/main.go Normal file
View File

@@ -0,0 +1,63 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"sort"
"github.com/AudriusButkevicius/cli"
)
type ByAlphabet []cli.Command
func (a ByAlphabet) Len() int { return len(a) }
func (a ByAlphabet) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByAlphabet) Less(i, j int) bool { return a[i].Name < a[j].Name }
var cliCommands []cli.Command
func main() {
app := cli.NewApp()
app.Name = "syncthing-cli"
app.Author = "Audrius Butkevičius"
app.Email = "audrius.butkevicius@gmail.com"
app.Usage = "Syncthing command line interface"
app.Version = "0.1"
app.HideHelp = true
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "endpoint, e",
Value: "http://127.0.0.1:8384",
Usage: "End point to connect to",
EnvVar: "STENDPOINT",
},
cli.StringFlag{
Name: "apikey, k",
Value: "",
Usage: "API Key",
EnvVar: "STAPIKEY",
},
cli.StringFlag{
Name: "username, u",
Value: "",
Usage: "Username",
EnvVar: "STUSERNAME",
},
cli.StringFlag{
Name: "password, p",
Value: "",
Usage: "Password",
EnvVar: "STPASSWORD",
},
cli.BoolFlag{
Name: "insecure, i",
Usage: "Do not verify SSL certificate",
EnvVar: "STINSECURE",
},
}
sort.Sort(ByAlphabet(cliCommands))
app.Commands = cliCommands
app.RunAndExitOnError()
}

165
cmd/stcli/utils.go Normal file
View File

@@ -0,0 +1,165 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"os"
"regexp"
"sort"
"strconv"
"strings"
"text/tabwriter"
"unicode"
"github.com/AudriusButkevicius/cli"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/protocol"
)
func responseToBArray(response *http.Response) []byte {
defer response.Body.Close()
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
die(err)
}
return bytes
}
func die(vals ...interface{}) {
if len(vals) > 1 || vals[0] != nil {
os.Stderr.WriteString(fmt.Sprintln(vals...))
os.Exit(1)
}
}
func wrappedHTTPPost(url string) func(c *cli.Context) {
return func(c *cli.Context) {
httpPost(c, url, "")
}
}
func prettyPrintJSON(json map[string]interface{}) {
writer := newTableWriter()
remap := make(map[string]interface{})
for k, v := range json {
key, ok := jsonAttributeLabels[k]
if !ok {
key = firstUpper(k)
}
remap[key] = v
}
jsonKeys := make([]string, 0, len(remap))
for key := range remap {
jsonKeys = append(jsonKeys, key)
}
sort.Strings(jsonKeys)
for _, k := range jsonKeys {
var value string
rvalue := remap[k]
switch rvalue.(type) {
case int, int16, int32, int64, uint, uint16, uint32, uint64, float32, float64:
value = fmt.Sprintf("%.0f", rvalue)
default:
value = fmt.Sprint(rvalue)
}
if value == "" {
continue
}
fmt.Fprintln(writer, k+":\t"+value)
}
writer.Flush()
}
func firstUpper(str string) string {
for i, v := range str {
return string(unicode.ToUpper(v)) + str[i+1:]
}
return ""
}
func newTableWriter() *tabwriter.Writer {
writer := new(tabwriter.Writer)
writer.Init(os.Stdout, 0, 8, 0, '\t', 0)
return writer
}
func getMyID(c *cli.Context) string {
response := httpGet(c, "system/status")
data := make(map[string]interface{})
json.Unmarshal(responseToBArray(response), &data)
return data["myID"].(string)
}
func getConfig(c *cli.Context) config.Configuration {
response := httpGet(c, "system/config")
config := config.Configuration{}
json.Unmarshal(responseToBArray(response), &config)
return config
}
func setConfig(c *cli.Context, cfg config.Configuration) {
body, err := json.Marshal(cfg)
die(err)
response := httpPost(c, "system/config", string(body))
if response.StatusCode != 200 {
die("Unexpected status code", response.StatusCode)
}
}
func parseBool(input string) bool {
val, err := strconv.ParseBool(input)
if err != nil {
die(input + " is not a valid value for a boolean")
}
return val
}
func parseInt(input string) int {
val, err := strconv.ParseInt(input, 0, 64)
if err != nil {
die(input + " is not a valid value for an integer")
}
return int(val)
}
func parseUint(input string) int {
val, err := strconv.ParseUint(input, 0, 64)
if err != nil {
die(input + " is not a valid value for an unsigned integer")
}
return int(val)
}
func parsePort(input string) int {
port := parseUint(input)
if port < 1 || port > 65535 {
die(input + " is not a valid port\nExpected value between 1 and 65535")
}
return port
}
func validAddress(input string) {
tokens := strings.Split(input, ":")
if len(tokens) != 2 {
die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
}
matched, err := regexp.MatchString("^[a-zA-Z0-9]+([-a-zA-Z0-9.]+[-a-zA-Z0-9]+)?$", tokens[0])
die(err)
if !matched {
die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
}
parsePort(tokens[1])
}
func parseDeviceID(input string) protocol.DeviceID {
device, err := protocol.DeviceIDFromString(input)
if err != nil {
die(input + " is not a valid device id")
}
return device
}

View File

@@ -7,6 +7,7 @@
package main
import (
"crypto/md5"
"errors"
"flag"
"fmt"
@@ -14,9 +15,6 @@ import (
"log"
"os"
"path/filepath"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/sha256"
)
func main() {
@@ -62,7 +60,7 @@ func compareDirectories(dirs ...string) error {
} else if res[i].name > res[0].name {
return fmt.Errorf("%s missing %v (present in %s)", dirs[i], res[0], dirs[0])
}
return fmt.Errorf("mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
return fmt.Errorf("Mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
}
}
@@ -76,7 +74,7 @@ type fileInfo struct {
name string
mode os.FileMode
mod int64
hash [sha256.Size]byte
hash [16]byte
}
func (f fileInfo) String() string {
@@ -90,10 +88,10 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
}
rn, _ := filepath.Rel(dir, path)
if rn == "." {
if rn == "." || rn == ".stfolder" {
return nil
}
if rn == ".stversions" || rn == ".stfolder" {
if rn == ".stversions" {
return filepath.SkipDir
}
@@ -108,7 +106,11 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
if err != nil {
return err
}
f.hash = sha256.Sum256([]byte(tgt))
h := md5.New()
h.Write([]byte(tgt))
hash := h.Sum(nil)
copy(f.hash[:], hash)
} else if info.IsDir() {
f = fileInfo{
name: rn,
@@ -121,7 +123,7 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
mode: info.Mode(),
mod: info.ModTime().Unix(),
}
sum, err := sha256file(path)
sum, err := md5file(path)
if err != nil {
return err
}
@@ -148,14 +150,14 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
return errc
}
func sha256file(fname string) (hash [sha256.Size]byte, err error) {
func md5file(fname string) (hash [16]byte, err error) {
f, err := os.Open(fname)
if err != nil {
return
}
defer f.Close()
h := sha256.New()
h := md5.New()
io.Copy(h, f)
hb := h.Sum(nil)
copy(hash[:], hb)

View File

File diff suppressed because it is too large Load Diff

View File

@@ -1,198 +0,0 @@
// Copyright (C) 2023 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"compress/gzip"
"context"
"io"
"log"
"math"
"os"
"path/filepath"
"sort"
"time"
)
type diskStore struct {
dir string
inbox chan diskEntry
maxBytes int64
maxFiles int
currentFiles []currentFile
currentSize int64
}
type diskEntry struct {
path string
data []byte
}
type currentFile struct {
path string
size int64
mtime int64
}
func (d *diskStore) Serve(ctx context.Context) {
if err := os.MkdirAll(d.dir, 0o700); err != nil {
log.Println("Creating directory:", err)
return
}
if err := d.inventory(); err != nil {
log.Println("Failed to inventory disk store:", err)
}
d.clean()
cleanTimer := time.NewTicker(time.Minute)
inventoryTimer := time.NewTicker(24 * time.Hour)
buf := new(bytes.Buffer)
gw := gzip.NewWriter(buf)
for {
select {
case entry := <-d.inbox:
path := d.fullPath(entry.path)
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
log.Println("Creating directory:", err)
continue
}
buf.Reset()
gw.Reset(buf)
if _, err := gw.Write(entry.data); err != nil {
log.Println("Failed to compress crash report:", err)
continue
}
if err := gw.Close(); err != nil {
log.Println("Failed to compress crash report:", err)
continue
}
if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
log.Printf("Failed to write %s: %v", entry.path, err)
_ = os.Remove(path)
continue
}
d.currentSize += int64(buf.Len())
d.currentFiles = append(d.currentFiles, currentFile{
size: int64(len(entry.data)),
path: path,
})
case <-cleanTimer.C:
d.clean()
case <-inventoryTimer.C:
if err := d.inventory(); err != nil {
log.Println("Failed to inventory disk store:", err)
}
case <-ctx.Done():
return
}
}
}
func (d *diskStore) Put(path string, data []byte) bool {
select {
case d.inbox <- diskEntry{
path: path,
data: data,
}:
return true
default:
return false
}
}
func (d *diskStore) Get(path string) ([]byte, error) {
path = d.fullPath(path)
bs, err := os.ReadFile(path)
if err != nil {
return nil, err
}
gr, err := gzip.NewReader(bytes.NewReader(bs))
if err != nil {
return nil, err
}
defer gr.Close()
return io.ReadAll(gr)
}
func (d *diskStore) Exists(path string) bool {
path = d.fullPath(path)
_, err := os.Lstat(path)
return err == nil
}
func (d *diskStore) clean() {
for len(d.currentFiles) > 0 && (len(d.currentFiles) > d.maxFiles || d.currentSize > d.maxBytes) {
f := d.currentFiles[0]
log.Println("Removing", f.path)
if err := os.Remove(f.path); err != nil {
log.Println("Failed to remove file:", err)
}
d.currentFiles = d.currentFiles[1:]
d.currentSize -= f.size
}
var oldest time.Duration
if len(d.currentFiles) > 0 {
oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
}
metricDiskstoreFilesTotal.Set(float64(len(d.currentFiles)))
metricDiskstoreBytesTotal.Set(float64(d.currentSize))
metricDiskstoreOldestAgeSeconds.Set(math.Round(oldest.Seconds()))
log.Printf("Clean complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
}
func (d *diskStore) inventory() error {
d.currentFiles = nil
d.currentSize = 0
err := filepath.Walk(d.dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
if filepath.Ext(path) != ".gz" {
return nil
}
d.currentSize += info.Size()
d.currentFiles = append(d.currentFiles, currentFile{
path: path,
size: info.Size(),
mtime: info.ModTime().Unix(),
})
return nil
})
sort.Slice(d.currentFiles, func(i, j int) bool {
return d.currentFiles[i].mtime < d.currentFiles[j].mtime
})
var oldest time.Duration
if len(d.currentFiles) > 0 {
oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
}
metricDiskstoreFilesTotal.Set(float64(len(d.currentFiles)))
metricDiskstoreBytesTotal.Set(float64(d.currentSize))
metricDiskstoreOldestAgeSeconds.Set(math.Round(oldest.Seconds()))
log.Printf("Inventory complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
return err
}
func (d *diskStore) fullPath(path string) string {
return filepath.Join(d.dir, path[0:2], path[2:]) + ".gz"
}

View File

@@ -1,160 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Command stcrashreceiver is a trivial HTTP server that allows two things:
//
// - uploading files (crash reports) named like a SHA256 hash using a PUT request
// - checking whether such file exists using a HEAD request
//
// Typically this should be deployed behind something that manages HTTPS.
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"github.com/alecthomas/kong"
raven "github.com/getsentry/raven-go"
"github.com/prometheus/client_golang/prometheus/promhttp"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/ur"
)
const maxRequestSize = 1 << 20 // 1 MiB
type cli struct {
Dir string `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."`
DSN string `help:"Sentry DSN" env:"SENTRY_DSN"`
Listen string `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"`
MaxDiskFiles int `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"`
MaxDiskSizeMB int64 `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"`
SentryQueue int `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"`
DiskQueue int `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"`
}
func main() {
var params cli
kong.Parse(&params)
mux := http.NewServeMux()
ds := &diskStore{
dir: filepath.Join(params.Dir, "crash_reports"),
inbox: make(chan diskEntry, params.DiskQueue),
maxFiles: params.MaxDiskFiles,
maxBytes: params.MaxDiskSizeMB << 20,
}
go ds.Serve(context.Background())
ss := &sentryService{
dsn: params.DSN,
inbox: make(chan sentryRequest, params.SentryQueue),
}
go ss.Serve(context.Background())
cr := &crashReceiver{
store: ds,
sentry: ss,
}
mux.Handle("/", cr)
mux.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) {
w.Write([]byte("OK"))
})
mux.Handle("/metrics", promhttp.Handler())
if params.DSN != "" {
mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports")))
}
log.SetOutput(os.Stdout)
if err := http.ListenAndServe(params.Listen, mux); err != nil {
log.Fatalln("HTTP serve:", err)
}
}
func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *http.Request) {
return func(w http.ResponseWriter, req *http.Request) {
result := "failure"
defer func() {
metricFailureReportsTotal.WithLabelValues(result).Inc()
}()
lr := io.LimitReader(req.Body, maxRequestSize)
bs, err := io.ReadAll(lr)
req.Body.Close()
if err != nil {
http.Error(w, err.Error(), 500)
return
}
var reports []ur.FailureReport
err = json.Unmarshal(bs, &reports)
if err != nil {
http.Error(w, err.Error(), 400)
return
}
if len(reports) == 0 {
// Shouldn't happen
log.Printf("Got zero failure reports")
return
}
version, err := parseVersion(reports[0].Version)
if err != nil {
http.Error(w, err.Error(), 400)
return
}
for _, r := range reports {
pkt := packet(version, "failure")
pkt.Message = r.Description
pkt.Extra = raven.Extra{
"count": r.Count,
}
for k, v := range r.Extra {
pkt.Extra[k] = v
}
if r.Goroutines != "" {
url, err := saveFailureWithGoroutines(r.FailureData, failureDir)
if err != nil {
log.Println("Saving failure report:", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
pkt.Extra["goroutinesURL"] = url
}
message := sanitizeMessageLDB(r.Description)
pkt.Fingerprint = []string{message}
if err := sendReport(dsn, pkt, userIDFor(req)); err != nil {
log.Println("Failed to send failure report:", err)
} else {
log.Println("Sent failure report:", r.Description)
result = "success"
}
}
}
}
func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string, error) {
bs := make([]byte, len(data.Description)+len(data.Goroutines))
copy(bs, data.Description)
copy(bs[len(data.Description):], data.Goroutines)
id := fmt.Sprintf("%x", sha256.Sum256(bs))
path := fullPathCompressed(failureDir, id)
err := compressAndWrite(bs, path)
if err != nil {
return "", err
}
return reportServer + path, nil
}

View File

@@ -1,40 +0,0 @@
// Copyright (C) 2023 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
metricCrashReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "crashreceiver",
Name: "crash_reports_total",
}, []string{"result"})
metricFailureReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "crashreceiver",
Name: "failure_reports_total",
}, []string{"result"})
metricDiskstoreFilesTotal = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: "syncthing",
Subsystem: "crashreceiver",
Name: "diskstore_files_total",
})
metricDiskstoreBytesTotal = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: "syncthing",
Subsystem: "crashreceiver",
Name: "diskstore_bytes_total",
})
metricDiskstoreOldestAgeSeconds = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: "syncthing",
Subsystem: "crashreceiver",
Name: "diskstore_oldest_age_seconds",
})
)

View File

@@ -1,321 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"context"
"errors"
"io"
"log"
"regexp"
"strings"
"sync"
raven "github.com/getsentry/raven-go"
"github.com/maruel/panicparse/v2/stack"
)
const reportServer = "https://crash.syncthing.net/report/"
var loader = newGithubSourceCodeLoader()
func init() {
raven.SetSourceCodeLoader(loader)
}
var (
clients = make(map[string]*raven.Client)
clientsMut sync.Mutex
)
type sentryService struct {
dsn string
inbox chan sentryRequest
}
type sentryRequest struct {
reportID string
userID string
data []byte
}
func (s *sentryService) Serve(ctx context.Context) {
for {
select {
case req := <-s.inbox:
pkt, err := parseCrashReport(req.reportID, req.data)
if err != nil {
log.Println("Failed to parse crash report:", err)
continue
}
if err := sendReport(s.dsn, pkt, req.userID); err != nil {
log.Println("Failed to send crash report:", err)
}
case <-ctx.Done():
return
}
}
}
func (s *sentryService) Send(reportID, userID string, data []byte) bool {
select {
case s.inbox <- sentryRequest{reportID, userID, data}:
return true
default:
return false
}
}
func sendReport(dsn string, pkt *raven.Packet, userID string) error {
pkt.Interfaces = append(pkt.Interfaces, &raven.User{ID: userID})
clientsMut.Lock()
defer clientsMut.Unlock()
cli, ok := clients[dsn]
if !ok {
var err error
cli, err = raven.New(dsn)
if err != nil {
return err
}
clients[dsn] = cli
}
// The client sets release and such on the packet before sending, in the
// misguided idea that it knows this better than than the packet we give
// it. So we copy the values from the packet to the client first...
cli.SetRelease(pkt.Release)
cli.SetEnvironment(pkt.Environment)
defer cli.Wait()
_, errC := cli.Capture(pkt, nil)
return <-errC
}
func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
parts := bytes.SplitN(report, []byte("\n"), 2)
if len(parts) != 2 {
return nil, errors.New("no first line")
}
version, err := parseVersion(string(parts[0]))
if err != nil {
return nil, err
}
report = parts[1]
foundPanic := false
var subjectLine []byte
for {
parts = bytes.SplitN(report, []byte("\n"), 2)
if len(parts) != 2 {
return nil, errors.New("no panic line found")
}
line := parts[0]
report = parts[1]
if foundPanic {
// The previous line was our "Panic at ..." header. We are now
// at the beginning of the real panic trace and this is our
// subject line.
subjectLine = line
break
} else if bytes.HasPrefix(line, []byte("Panic at")) {
foundPanic = true
}
}
r := bytes.NewReader(report)
ctx, _, err := stack.ScanSnapshot(r, io.Discard, stack.DefaultOpts())
if err != nil && err != io.EOF {
return nil, err
}
if ctx == nil || len(ctx.Goroutines) == 0 {
return nil, errors.New("no goroutines found")
}
// Lock the source code loader to the version we are processing here.
if version.commit != "" {
// We have a commit hash, so we know exactly which source to use
loader.LockWithVersion(version.commit)
} else if strings.HasPrefix(version.tag, "v") {
// Lets hope the tag is close enough
loader.LockWithVersion(version.tag)
} else {
// Last resort
loader.LockWithVersion("main")
}
defer loader.Unlock()
var trace raven.Stacktrace
for _, gr := range ctx.Goroutines {
if gr.First {
trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
for i, sc := range gr.Stack.Calls {
trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name, sc.RemoteSrcPath, sc.Line, 3, nil)
}
break
}
}
pkt := packet(version, "crash")
pkt.Message = string(subjectLine)
pkt.Extra = raven.Extra{
"url": reportServer + path,
}
pkt.Interfaces = []raven.Interface{&trace}
pkt.Fingerprint = crashReportFingerprint(pkt.Message)
return pkt, nil
}
var (
indexRe = regexp.MustCompile(`\[[-:0-9]+\]`)
sizeRe = regexp.MustCompile(`(length|capacity) [0-9]+`)
ldbPosRe = regexp.MustCompile(`(\(pos=)([0-9]+)\)`)
ldbChecksumRe = regexp.MustCompile(`(want=0x)([a-z0-9]+)( got=0x)([a-z0-9]+)`)
ldbFileRe = regexp.MustCompile(`(\[file=)([0-9]+)(\.ldb\])`)
ldbInternalKeyRe = regexp.MustCompile(`(internal key ")[^"]+(", len=)[0-9]+`)
ldbPathRe = regexp.MustCompile(`(open|write|read) .+[\\/].+[\\/]index[^\\/]+[\\/][^\\/]+: `)
)
func sanitizeMessageLDB(message string) string {
message = ldbPosRe.ReplaceAllString(message, "${1}x)")
message = ldbFileRe.ReplaceAllString(message, "${1}x${3}")
message = ldbChecksumRe.ReplaceAllString(message, "${1}X${3}X")
message = ldbInternalKeyRe.ReplaceAllString(message, "${1}x${2}x")
message = ldbPathRe.ReplaceAllString(message, "$1 x: ")
return message
}
func crashReportFingerprint(message string) []string {
// Do not fingerprint on the stack in case of db corruption or fatal
// db io error - where it occurs doesn't matter.
orig := message
message = sanitizeMessageLDB(message)
if message != orig {
return []string{message}
}
message = indexRe.ReplaceAllString(message, "[x]")
message = sizeRe.ReplaceAllString(message, "$1 x")
// {{ default }} is what sentry uses as a fingerprint by default. While
// never specified, the docs point at this being some hash derived from the
// stack trace. Here we include the filtered panic message on top of that.
// https://docs.sentry.io/platforms/go/data-management/event-grouping/sdk-fingerprinting/#basic-example
return []string{"{{ default }}", message}
}
// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]
// or, somewhere along the way the "+" in the version tag disappeared:
// syncthing v1.23.7-dev.26.gdf7b56ae.dirty-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]
var (
longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
gitExtraRE = regexp.MustCompile(`\.\d+\.g[0-9a-f]+`) // ".1.g6aaae618"
gitExtraSepRE = regexp.MustCompile(`[.-]`) // dot or dash
)
type version struct {
version string // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
tag string // "v1.1.4-rc.1"
commit string // "6aaae618", blank when absent
codename string // "Erbium Earthworm"
runtime string // "go1.12.5"
goos string // "darwin"
goarch string // "amd64"
builder string // "jb@kvin.kastelo.net"
extra []string // "foo", "bar"
}
func (v version) environment() string {
if v.commit != "" {
return "Development"
}
if strings.Contains(v.tag, "-rc.") {
return "Candidate"
}
if strings.Contains(v.tag, "-") {
return "Beta"
}
return "Stable"
}
func parseVersion(line string) (version, error) {
m := longVersionRE.FindStringSubmatch(line)
if len(m) == 0 {
return version{}, errors.New("unintelligeble version string")
}
v := version{
version: m[1],
codename: m[2],
runtime: m[3],
goos: m[4],
goarch: m[5],
builder: m[6],
}
// Split the version tag into tag and commit. This is old style
// v1.2.3-something.4+11-g12345678 or newer with just dots
// v1.2.3-something.4.11.g12345678 or v1.2.3-dev.11.g12345678.
parts := []string{v.version}
if strings.Contains(v.version, "+") {
parts = strings.Split(v.version, "+")
} else {
idxs := gitExtraRE.FindStringIndex(v.version)
if len(idxs) > 0 {
parts = []string{v.version[:idxs[0]], v.version[idxs[0]+1:]}
}
}
v.tag = parts[0]
if len(parts) > 1 {
fields := gitExtraSepRE.Split(parts[1], -1)
if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
v.commit = fields[1][1:]
}
}
if len(m) >= 8 && m[7] != "" {
tags := strings.Split(m[7], ",")
for i := range tags {
tags[i] = strings.TrimSpace(tags[i])
}
v.extra = tags
}
return v, nil
}
func packet(version version, reportType string) *raven.Packet {
pkt := &raven.Packet{
Platform: "go",
Release: version.tag,
Environment: version.environment(),
Tags: raven.Tags{
raven.Tag{Key: "version", Value: version.version},
raven.Tag{Key: "tag", Value: version.tag},
raven.Tag{Key: "codename", Value: version.codename},
raven.Tag{Key: "runtime", Value: version.runtime},
raven.Tag{Key: "goos", Value: version.goos},
raven.Tag{Key: "goarch", Value: version.goarch},
raven.Tag{Key: "builder", Value: version.builder},
raven.Tag{Key: "report_type", Value: reportType},
},
}
if version.commit != "" {
pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
}
for _, tag := range version.extra {
pkt.Tags = append(pkt.Tags, raven.Tag{Key: tag, Value: "1"})
}
return pkt
}

View File

@@ -1,155 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"fmt"
"os"
"testing"
)
func TestParseVersion(t *testing.T) {
cases := []struct {
longVersion string
parsed version
}{
{
longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`,
parsed: version{
version: "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
tag: "v1.1.4-rc.1",
commit: "6aaae618",
codename: "Erbium Earthworm",
runtime: "go1.12.5",
goos: "darwin",
goarch: "amd64",
builder: "jb@kvin.kastelo.net",
},
},
{
longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]`,
parsed: version{
version: "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
tag: "v1.1.4-rc.1",
commit: "6aaae618",
codename: "Erbium Earthworm",
runtime: "go1.12.5",
goos: "darwin",
goarch: "amd64",
builder: "jb@kvin.kastelo.net",
extra: []string{"foo", "bar"},
},
},
{
longVersion: `syncthing v1.23.7-dev.26.gdf7b56ae-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]`,
parsed: version{
version: "v1.23.7-dev.26.gdf7b56ae-stversionextra",
tag: "v1.23.7-dev",
commit: "df7b56ae",
codename: "Fermium Flea",
runtime: "go1.20.5",
goos: "darwin",
goarch: "arm64",
builder: "jb@ok.kastelo.net",
extra: []string{"Some Wrapper", "purego", "stnoupgrade"},
},
},
}
for _, tc := range cases {
v, err := parseVersion(tc.longVersion)
if err != nil {
t.Errorf("%s\nerror: %v\n", tc.longVersion, err)
continue
}
if fmt.Sprint(v) != fmt.Sprint(tc.parsed) {
t.Errorf("%s\nA: %v\nE: %v\n", tc.longVersion, v, tc.parsed)
}
}
}
func TestParseReport(t *testing.T) {
bs, err := os.ReadFile("_testdata/panic.log")
if err != nil {
t.Fatal(err)
}
pkt, err := parseCrashReport("1/2/345", bs)
if err != nil {
t.Fatal(err)
}
bs, err = pkt.JSON()
if err != nil {
t.Fatal(err)
}
fmt.Printf("%s\n", bs)
}
func TestCrashReportFingerprint(t *testing.T) {
cases := []struct {
message, exp string
ldb bool
}{
{
message: "panic: leveldb/table: corruption on data-block (pos=51308946): checksum mismatch, want=0xa89f9aa0 got=0xd27cc4c7 [file=004003.ldb]",
exp: "panic: leveldb/table: corruption on data-block (pos=x): checksum mismatch, want=0xX got=0xX [file=x.ldb]",
ldb: true,
},
{
message: "panic: leveldb/table: corruption on table-footer (pos=248): bad magic number [file=001370.ldb]",
exp: "panic: leveldb/table: corruption on table-footer (pos=x): bad magic number [file=x.ldb]",
ldb: true,
},
{
message: "panic: runtime error: slice bounds out of range [4294967283:4194304]",
exp: "panic: runtime error: slice bounds out of range [x]",
},
{
message: "panic: runtime error: slice bounds out of range [-2:]",
exp: "panic: runtime error: slice bounds out of range [x]",
},
{
message: "panic: runtime error: slice bounds out of range [:4294967283] with capacity 32768",
exp: "panic: runtime error: slice bounds out of range [x] with capacity x",
},
{
message: "panic: runtime error: index out of range [0] with length 0",
exp: "panic: runtime error: index out of range [x] with length x",
},
{
message: `panic: leveldb: internal key "\x01", len=1: invalid length`,
exp: `panic: leveldb: internal key "x", len=x: invalid length`,
ldb: true,
},
{
message: `panic: write /var/syncthing/config/index-v0.14.0.db/2732813.log: cannot allocate memory`,
exp: `panic: write x: cannot allocate memory`,
ldb: true,
},
{
message: `panic: filling Blocks: read C:\Users\Serv-Resp-Tizayuca\AppData\Local\Syncthing\index-v0.14.0.db\006561.ldb: Error de datos (comprobación de redundancia cíclica).`,
exp: `panic: filling Blocks: read x: Error de datos (comprobación de redundancia cíclica).`,
ldb: true,
},
}
for i, tc := range cases {
fingerprint := crashReportFingerprint(tc.message)
expLen := 2
if tc.ldb {
expLen = 1
}
if l := len(fingerprint); l != expLen {
t.Errorf("tc %v: Unexpected fingerprint length: %v != %v", i, l, expLen)
} else if msg := fingerprint[expLen-1]; msg != tc.exp {
t.Errorf("tc %v:\n\"%v\" !=\n\"%v\"", i, msg, tc.exp)
}
}
}

View File

@@ -1,118 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"fmt"
"io"
"net/http"
"path/filepath"
"strings"
"sync"
"time"
)
const (
urlPrefix = "https://raw.githubusercontent.com/syncthing/syncthing/"
httpTimeout = 10 * time.Second
)
type githubSourceCodeLoader struct {
mut sync.Mutex
version string
cache map[string]map[string][][]byte // version -> file -> lines
client *http.Client
}
func newGithubSourceCodeLoader() *githubSourceCodeLoader {
return &githubSourceCodeLoader{
cache: make(map[string]map[string][][]byte),
client: &http.Client{Timeout: httpTimeout},
}
}
func (l *githubSourceCodeLoader) LockWithVersion(version string) {
l.mut.Lock()
l.version = version
if _, ok := l.cache[version]; !ok {
l.cache[version] = make(map[string][][]byte)
}
}
func (l *githubSourceCodeLoader) Unlock() {
l.mut.Unlock()
}
func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]byte, int) {
filename = filepath.ToSlash(filename)
lines, ok := l.cache[l.version][filename]
if !ok {
// Cache whatever we managed to find (or nil if nothing, so we don't try again)
defer func() {
l.cache[l.version][filename] = lines
}()
knownPrefixes := []string{"/lib/", "/cmd/"}
var idx int
for _, pref := range knownPrefixes {
idx = strings.Index(filename, pref)
if idx >= 0 {
break
}
}
if idx == -1 {
return nil, 0
}
url := urlPrefix + l.version + filename[idx:]
resp, err := l.client.Get(url)
if err != nil {
fmt.Println("Loading source:", err)
return nil, 0
}
if resp.StatusCode != http.StatusOK {
fmt.Println("Loading source:", resp.Status)
return nil, 0
}
data, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
fmt.Println("Loading source:", err.Error())
return nil, 0
}
lines = bytes.Split(data, []byte{'\n'})
}
return getLineFromLines(lines, line, context)
}
func getLineFromLines(lines [][]byte, line, context int) ([][]byte, int) {
if lines == nil {
// cached error from ReadFile: return no lines
return nil, 0
}
line-- // stack trace lines are 1-indexed
start := line - context
var idx int
if start < 0 {
start = 0
idx = line
} else {
idx = context
}
end := line + context + 1
if line >= len(lines) {
return nil, 0
}
if end > len(lines) {
end = len(lines)
}
return lines[start:end], idx
}

View File

@@ -1,102 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"io"
"log"
"net/http"
"path"
"strings"
)
type crashReceiver struct {
store *diskStore
sentry *sentryService
}
func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// The final path component should be a SHA256 hash in hex, so 64 hex
// characters. We don't care about case on the request but use lower
// case internally.
reportID := strings.ToLower(path.Base(req.URL.Path))
if len(reportID) != 64 {
http.Error(w, "Bad request", http.StatusBadRequest)
return
}
for _, c := range reportID {
if c >= 'a' && c <= 'f' {
continue
}
if c >= '0' && c <= '9' {
continue
}
http.Error(w, "Bad request", http.StatusBadRequest)
return
}
switch req.Method {
case http.MethodGet:
r.serveGet(reportID, w, req)
case http.MethodHead:
r.serveHead(reportID, w, req)
case http.MethodPut:
r.servePut(reportID, w, req)
default:
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
}
}
// serveGet responds to GET requests by serving the uncompressed report.
func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http.Request) {
bs, err := r.store.Get(reportID)
if err != nil {
http.Error(w, "Not found", http.StatusNotFound)
return
}
w.Write(bs)
}
// serveHead responds to HEAD requests by checking if the named report
// already exists in the system.
func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) {
if !r.store.Exists(reportID) {
http.Error(w, "Not found", http.StatusNotFound)
}
}
// servePut accepts and stores the given report.
func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *http.Request) {
result := "receive_failure"
defer func() {
metricCrashReportsTotal.WithLabelValues(result).Inc()
}()
// Read at most maxRequestSize of report data.
log.Println("Receiving report", reportID)
lr := io.LimitReader(req.Body, maxRequestSize)
bs, err := io.ReadAll(lr)
if err != nil {
log.Println("Reading report:", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
result = "success"
// Store the report
if !r.store.Put(reportID, bs) {
log.Println("Failed to store report (queue full):", reportID)
result = "queue_failure"
}
// Send the report to Sentry
if !r.sentry.Send(reportID, userIDFor(req), bs) {
log.Println("Failed to send report to sentry (queue full):", reportID)
result = "sentry_failure"
}
}

View File

@@ -1,57 +0,0 @@
// Copyright (C) 2021 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"compress/gzip"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"time"
"github.com/syncthing/syncthing/lib/sha256"
)
// userIDFor returns a string we can use as the user ID for the purpose of
// counting affected users. It's the truncated hash of a salt, the user
// remote IP, and the current month.
func userIDFor(req *http.Request) string {
addr := req.RemoteAddr
if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
addr = fwd
}
if host, _, err := net.SplitHostPort(addr); err == nil {
addr = host
}
now := time.Now().Format("200601")
salt := "stcrashreporter"
hash := sha256.Sum256([]byte(salt + addr + now))
return fmt.Sprintf("%x", hash[:8])
}
// 01234567890abcdef... => 01/23
func dirFor(base string) string {
return filepath.Join(base[0:2], base[2:4])
}
func fullPathCompressed(root, reportID string) string {
return filepath.Join(root, dirFor(reportID), reportID) + ".gz"
}
func compressAndWrite(bs []byte, fullPath string) error {
// Compress the report for storage
buf := new(bytes.Buffer)
gw := gzip.NewWriter(buf)
_, _ = gw.Write(bs) // can't fail
gw.Close()
// Create an output file with the compressed report
return os.WriteFile(fullPath, buf.Bytes(), 0644)
}

View File

@@ -7,7 +7,6 @@
package main
import (
"context"
"crypto/rand"
"encoding/binary"
"flag"
@@ -15,7 +14,6 @@ import (
"strings"
"time"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/beacon"
"github.com/syncthing/syncthing/lib/discover"
"github.com/syncthing/syncthing/lib/protocol"
@@ -49,16 +47,14 @@ func main() {
log.Println("My ID:", myID)
}
ctx := context.Background()
runbeacon(ctx, beacon.NewMulticast(mc), fake)
runbeacon(ctx, beacon.NewBroadcast(bc), fake)
runbeacon(beacon.NewMulticast(mc), fake)
runbeacon(beacon.NewBroadcast(bc), fake)
select {}
}
func runbeacon(ctx context.Context, bc beacon.Interface, fake bool) {
go bc.Serve(ctx)
func runbeacon(bc beacon.Interface, fake bool) {
go bc.Serve()
go recv(bc)
if fake {
go send(bc)

19
cmd/stdiscosrv/LICENSE Normal file
View File

@@ -0,0 +1,19 @@
Copyright (C) 2014-2015 The Discosrv Authors
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -6,5 +6,33 @@ This is the global discovery server for the `syncthing` project.
Usage
-----
https://docs.syncthing.net/users/stdiscosrv.html
The discovery server supports `ql` and `postgres` backends.
Specify the backend via `-db-backend` and the database DSN via `-db-dsn`.
By default it will use in-memory `ql` backend. If you wish to persist the
information on disk between restarts in `ql`, specify a file DSN:
```bash
$ stdiscosrv -db-dsn="file:///var/run/stdiscosrv.db"
```
For `postgres`, you will need to create a database and a user with permissions
to create tables in it, then start the stdiscosrv as follows:
```bash
$ export STDISCOSRV_DB_DSN="postgres://user:password@localhost/databasename"
$ stdiscosrv -db-backend="postgres"
```
You can pass the DSN as command line option, but the value what you pass in will
be visible in most process managers, potentially exposing the database password
to other users.
In all cases, the appropriate tables and indexes will be created at first
startup. If it doesn't exit with an error, you're fine.
See `stdiscosrv -help` for other options.
##### Third-party attribution
[cznic/lldb](https://github.com/cznic/lldb), Copyright (C) 2014 The lldb Authors.

View File

@@ -1,508 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"compress/gzip"
"context"
"crypto/tls"
"encoding/base64"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
io "io"
"log"
"math/rand"
"net"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/stringutil"
)
// announcement is the format received from and sent to clients
type announcement struct {
Seen time.Time `json:"seen"`
Addresses []string `json:"addresses"`
}
type apiSrv struct {
addr string
cert tls.Certificate
db database
listener net.Listener
repl replicator // optional
useHTTP bool
mapsMut sync.Mutex
misses map[string]int32
}
type requestID int64
func (i requestID) String() string {
return fmt.Sprintf("%016x", int64(i))
}
type contextKey int
const idKey contextKey = iota
func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP bool) *apiSrv {
return &apiSrv{
addr: addr,
cert: cert,
db: db,
repl: repl,
useHTTP: useHTTP,
misses: make(map[string]int32),
}
}
func (s *apiSrv) Serve(_ context.Context) error {
if s.useHTTP {
listener, err := net.Listen("tcp", s.addr)
if err != nil {
log.Println("Listen:", err)
return err
}
s.listener = listener
} else {
tlsCfg := &tls.Config{
Certificates: []tls.Certificate{s.cert},
ClientAuth: tls.RequestClientCert,
MinVersion: tls.VersionTLS12,
NextProtos: []string{"h2", "http/1.1"},
}
tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
if err != nil {
log.Println("Listen:", err)
return err
}
s.listener = tlsListener
}
http.HandleFunc("/", s.handler)
http.HandleFunc("/ping", handlePing)
srv := &http.Server{
ReadTimeout: httpReadTimeout,
WriteTimeout: httpWriteTimeout,
MaxHeaderBytes: httpMaxHeaderBytes,
ErrorLog: log.New(io.Discard, "", 0),
}
err := srv.Serve(s.listener)
if err != nil {
log.Println("Serve:", err)
}
return err
}
func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
t0 := time.Now()
lw := NewLoggingResponseWriter(w)
defer func() {
diff := time.Since(t0)
apiRequestsSeconds.WithLabelValues(req.Method).Observe(diff.Seconds())
apiRequestsTotal.WithLabelValues(req.Method, strconv.Itoa(lw.statusCode)).Inc()
}()
reqID := requestID(rand.Int63())
req = req.WithContext(context.WithValue(req.Context(), idKey, reqID))
if debug {
log.Println(reqID, req.Method, req.URL, req.Proto)
}
remoteAddr := &net.TCPAddr{
IP: nil,
Port: -1,
}
if s.useHTTP {
// X-Forwarded-For can have multiple client IPs; split using the comma separator
forwardIP, _, _ := strings.Cut(req.Header.Get("X-Forwarded-For"), ",")
// net.ParseIP will return nil if leading/trailing whitespace exists; use strings.TrimSpace()
remoteAddr.IP = net.ParseIP(strings.TrimSpace(forwardIP))
if parsedPort, err := strconv.ParseInt(req.Header.Get("X-Client-Port"), 10, 0); err == nil {
remoteAddr.Port = int(parsedPort)
}
} else {
var err error
remoteAddr, err = net.ResolveTCPAddr("tcp", req.RemoteAddr)
if err != nil {
log.Println("remoteAddr:", err)
lw.Header().Set("Retry-After", errorRetryAfterString())
http.Error(lw, "Internal Server Error", http.StatusInternalServerError)
apiRequestsTotal.WithLabelValues("no_remote_addr").Inc()
return
}
}
switch req.Method {
case http.MethodGet:
s.handleGET(lw, req)
case http.MethodPost:
s.handlePOST(remoteAddr, lw, req)
default:
http.Error(lw, "Method Not Allowed", http.StatusMethodNotAllowed)
}
}
func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
reqID := req.Context().Value(idKey).(requestID)
deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
if err != nil {
if debug {
log.Println(reqID, "bad device param")
}
lookupRequestsTotal.WithLabelValues("bad_request").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
key := deviceID.String()
rec, err := s.db.get(key)
if err != nil {
// some sort of internal error
lookupRequestsTotal.WithLabelValues("internal_error").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
if len(rec.Addresses) == 0 {
lookupRequestsTotal.WithLabelValues("not_found").Inc()
s.mapsMut.Lock()
misses := s.misses[key]
if misses < rec.Misses {
misses = rec.Misses + 1
} else {
misses++
}
s.misses[key] = misses
s.mapsMut.Unlock()
if misses%notFoundMissesWriteInterval == 0 {
rec.Misses = misses
rec.Missed = time.Now().UnixNano()
rec.Addresses = nil
// rec.Seen retained from get
s.db.put(key, rec)
}
afterS := notFoundRetryAfterSeconds(int(misses))
retryAfterHistogram.Observe(float64(afterS))
w.Header().Set("Retry-After", strconv.Itoa(afterS))
http.Error(w, "Not Found", http.StatusNotFound)
return
}
lookupRequestsTotal.WithLabelValues("success").Inc()
w.Header().Set("Content-Type", "application/json")
var bw io.Writer = w
// Use compression if the client asks for it
if strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
w.Header().Set("Content-Encoding", "gzip")
gw := gzip.NewWriter(bw)
defer gw.Close()
bw = gw
}
json.NewEncoder(bw).Encode(announcement{
Seen: time.Unix(0, rec.Seen).Truncate(time.Second),
Addresses: addressStrs(rec.Addresses),
})
}
func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req *http.Request) {
reqID := req.Context().Value(idKey).(requestID)
rawCert, err := certificateBytes(req)
if err != nil {
if debug {
log.Println(reqID, "no certificates:", err)
}
announceRequestsTotal.WithLabelValues("no_certificate").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Forbidden", http.StatusForbidden)
return
}
var ann announcement
if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
if debug {
log.Println(reqID, "decode:", err)
}
announceRequestsTotal.WithLabelValues("bad_request").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
deviceID := protocol.NewDeviceID(rawCert)
addresses := fixupAddresses(remoteAddr, ann.Addresses)
if len(addresses) == 0 {
announceRequestsTotal.WithLabelValues("bad_request").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
if err := s.handleAnnounce(deviceID, addresses); err != nil {
announceRequestsTotal.WithLabelValues("internal_error").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
announceRequestsTotal.WithLabelValues("success").Inc()
w.Header().Set("Reannounce-After", reannounceAfterString())
w.WriteHeader(http.StatusNoContent)
}
func (s *apiSrv) Stop() {
s.listener.Close()
}
func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string) error {
key := deviceID.String()
now := time.Now()
expire := now.Add(addressExpiryTime).UnixNano()
dbAddrs := make([]DatabaseAddress, len(addresses))
for i := range addresses {
dbAddrs[i].Address = addresses[i]
dbAddrs[i].Expires = expire
}
// The address slice must always be sorted for database merges to work
// properly.
sort.Sort(databaseAddressOrder(dbAddrs))
seen := now.UnixNano()
if s.repl != nil {
s.repl.send(key, dbAddrs, seen)
}
return s.db.merge(key, dbAddrs, seen)
}
func handlePing(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(204)
}
func certificateBytes(req *http.Request) ([]byte, error) {
if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
return req.TLS.PeerCertificates[0].Raw, nil
}
var bs []byte
if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
if strings.Contains(hdr, "%") {
// Nginx using $ssl_client_escaped_cert
// The certificate is in PEM format with url encoding.
// We need to decode for the PEM decoder
hdr, err := url.QueryUnescape(hdr)
if err != nil {
// Decoding failed
return nil, err
}
bs = []byte(hdr)
} else {
// Nginx using $ssl_client_cert
// The certificate is in PEM format but with spaces for newlines. We
// need to reinstate the newlines for the PEM decoder. But we need to
// leave the spaces in the BEGIN and END lines - the first and last
// space - alone.
bs = []byte(hdr)
firstSpace := bytes.Index(bs, []byte(" "))
lastSpace := bytes.LastIndex(bs, []byte(" "))
for i := firstSpace + 1; i < lastSpace; i++ {
if bs[i] == ' ' {
bs[i] = '\n'
}
}
}
} else if hdr := req.Header.Get("X-Tls-Client-Cert-Der-Base64"); hdr != "" {
// Caddy {tls_client_certificate_der_base64}
hdr, err := base64.StdEncoding.DecodeString(hdr)
if err != nil {
// Decoding failed
return nil, err
}
bs = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: hdr})
} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
// Traefik 2 passtlsclientcert
//
// The certificate is in PEM format, maybe with URL encoding
// (depends on Traefik version) but without newlines and start/end
// statements. We need to decode, reinstate the newlines every 64
// character and add statements for the PEM decoder
if strings.Contains(hdr, "%") {
if unesc, err := url.QueryUnescape(hdr); err == nil {
hdr = unesc
}
}
for i := 64; i < len(hdr); i += 65 {
hdr = hdr[:i] + "\n" + hdr[i:]
}
hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
hdr += "\n-----END CERTIFICATE-----\n"
bs = []byte(hdr)
}
if bs == nil {
return nil, errors.New("empty certificate header")
}
block, _ := pem.Decode(bs)
if block == nil {
// Decoding failed
return nil, errors.New("certificate decode result is empty")
}
return block.Bytes, nil
}
// fixupAddresses checks the list of addresses, removing invalid ones and
// replacing unspecified IPs with the given remote IP.
func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
fixed := make([]string, 0, len(addresses))
for _, annAddr := range addresses {
uri, err := url.Parse(annAddr)
if err != nil {
continue
}
host, port, err := net.SplitHostPort(uri.Host)
if err != nil {
continue
}
ip := net.ParseIP(host)
// Some classes of IP are no-go.
if ip.IsLoopback() || ip.IsMulticast() {
continue
}
if host == "" || ip.IsUnspecified() {
if remote != nil {
// Replace the unspecified IP with the request source.
// ... unless the request source is the loopback address or
// multicast/unspecified (can't happen, really).
if remote.IP == nil || remote.IP.IsLoopback() || remote.IP.IsMulticast() || remote.IP.IsUnspecified() {
continue
}
// Do not use IPv6 remote address if requested scheme is ...4
// (i.e., tcp4, etc.)
if strings.HasSuffix(uri.Scheme, "4") && remote.IP.To4() == nil {
continue
}
// Do not use IPv4 remote address if requested scheme is ...6
if strings.HasSuffix(uri.Scheme, "6") && remote.IP.To4() != nil {
continue
}
host = remote.IP.String()
} else {
// remote is nil, unable to determine host IP
continue
}
}
// If zero port was specified, use remote port.
if port == "0" {
if remote != nil && remote.Port > 0 {
// use remote port
port = strconv.Itoa(remote.Port)
} else {
// unable to determine remote port
continue
}
}
uri.Host = net.JoinHostPort(host, port)
fixed = append(fixed, uri.String())
}
// Remove duplicate addresses
fixed = stringutil.UniqueTrimmedStrings(fixed)
return fixed
}
type loggingResponseWriter struct {
http.ResponseWriter
statusCode int
}
func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
return &loggingResponseWriter{w, http.StatusOK}
}
func (lrw *loggingResponseWriter) WriteHeader(code int) {
lrw.statusCode = code
lrw.ResponseWriter.WriteHeader(code)
}
func addressStrs(dbAddrs []DatabaseAddress) []string {
res := make([]string, len(dbAddrs))
for i, a := range dbAddrs {
res[i] = a.Address
}
return res
}
func errorRetryAfterString() string {
return strconv.Itoa(errorRetryAfterSeconds + rand.Intn(errorRetryFuzzSeconds))
}
func notFoundRetryAfterSeconds(misses int) int {
retryAfterS := notFoundRetryMinSeconds + notFoundRetryIncSeconds*misses
if retryAfterS > notFoundRetryMaxSeconds {
retryAfterS = notFoundRetryMaxSeconds
}
retryAfterS += rand.Intn(notFoundRetryFuzzSeconds)
return retryAfterS
}
func reannounceAfterString() string {
return strconv.Itoa(reannounceAfterSeconds + rand.Intn(reannounzeFuzzSeconds))
}

View File

@@ -1,96 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"fmt"
"net"
"testing"
)
func TestFixupAddresses(t *testing.T) {
cases := []struct {
remote *net.TCPAddr
in []string
out []string
}{
{ // verbatim passthrough
in: []string{"tcp://1.2.3.4:22000"},
out: []string{"tcp://1.2.3.4:22000"},
}, { // unspecified replaced by remote
remote: addr("1.2.3.4", 22000),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://1.2.3.4:22000", "tcp://192.0.2.42:22000"},
}, { // unspecified not used as replacement
remote: addr("0.0.0.0", 22000),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // unspecified not used as replacement
remote: addr("::", 22000),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // localhost not used as replacement
remote: addr("127.0.0.1", 22000),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // localhost not used as replacement
remote: addr("::1", 22000),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // multicast not used as replacement
remote: addr("224.0.0.1", 22000),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // multicast not used as replacement
remote: addr("ff80::42", 22000),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // explicitly announced weirdness is also filtered
remote: addr("192.0.2.42", 22000),
in: []string{"tcp://:22000", "tcp://127.1.2.3:22000", "tcp://[::1]:22000", "tcp://[ff80::42]:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // port remapping
remote: addr("123.123.123.123", 9000),
in: []string{"tcp://0.0.0.0:0"},
out: []string{"tcp://123.123.123.123:9000"},
}, { // unspecified port remapping
remote: addr("123.123.123.123", 9000),
in: []string{"tcp://:0"},
out: []string{"tcp://123.123.123.123:9000"},
}, { // empty remapping
remote: addr("123.123.123.123", 9000),
in: []string{"tcp://"},
out: []string{},
}, { // port only remapping
remote: addr("123.123.123.123", 9000),
in: []string{"tcp://44.44.44.44:0"},
out: []string{"tcp://44.44.44.44:9000"},
}, { // remote ip nil
remote: addr("", 9000),
in: []string{"tcp://:22000", "tcp://44.44.44.44:9000"},
out: []string{"tcp://44.44.44.44:9000"},
}, { // remote port 0
remote: addr("123.123.123.123", 0),
in: []string{"tcp://:22000", "tcp://44.44.44.44"},
out: []string{"tcp://123.123.123.123:22000"},
},
}
for _, tc := range cases {
out := fixupAddresses(tc.remote, tc.in)
if fmt.Sprint(out) != fmt.Sprint(tc.out) {
t.Errorf("fixupAddresses(%v, %v) => %v, expected %v", tc.remote, tc.in, out, tc.out)
}
}
}
func addr(host string, port int) *net.TCPAddr {
return &net.TCPAddr{
IP: net.ParseIP(host),
Port: port,
}
}

75
cmd/stdiscosrv/clean.go Normal file
View File

@@ -0,0 +1,75 @@
// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
package main
import (
"database/sql"
"log"
"time"
)
type cleansrv struct {
intv time.Duration
db *sql.DB
prep map[string]*sql.Stmt
}
func (s *cleansrv) Serve() {
for {
time.Sleep(next(s.intv))
err := s.cleanOldEntries()
if err != nil {
log.Println("Clean:", err)
}
}
}
func (s *cleansrv) Stop() {
panic("stop unimplemented")
}
func (s *cleansrv) cleanOldEntries() (err error) {
var tx *sql.Tx
tx, err = s.db.Begin()
if err != nil {
return err
}
defer func() {
if err == nil {
err = tx.Commit()
} else {
tx.Rollback()
}
}()
res, err := tx.Stmt(s.prep["cleanAddress"]).Exec()
if err != nil {
return err
}
if rows, _ := res.RowsAffected(); rows > 0 {
log.Printf("Clean: %d old addresses", rows)
}
res, err = tx.Stmt(s.prep["cleanDevice"]).Exec()
if err != nil {
return err
}
if rows, _ := res.RowsAffected(); rows > 0 {
log.Printf("Clean: %d old devices", rows)
}
var devs, addrs int
row := tx.Stmt(s.prep["countDevice"]).QueryRow()
if err = row.Scan(&devs); err != nil {
return err
}
row = tx.Stmt(s.prep["countAddress"]).QueryRow()
if err = row.Scan(&addrs); err != nil {
return err
}
log.Printf("Database: %d devices, %d addresses", devs, addrs)
return nil
}

View File

@@ -1,413 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
//go:generate go run ../../proto/scripts/protofmt.go database.proto
//go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
package main
import (
"context"
"log"
"net"
"net/url"
"sort"
"time"
"github.com/syncthing/syncthing/lib/sliceutil"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/storage"
"github.com/syndtr/goleveldb/leveldb/util"
)
type clock interface {
Now() time.Time
}
type defaultClock struct{}
func (defaultClock) Now() time.Time {
return time.Now()
}
type database interface {
put(key string, rec DatabaseRecord) error
merge(key string, addrs []DatabaseAddress, seen int64) error
get(key string) (DatabaseRecord, error)
}
type levelDBStore struct {
db *leveldb.DB
inbox chan func()
clock clock
marshalBuf []byte
}
func newLevelDBStore(dir string) (*levelDBStore, error) {
db, err := leveldb.OpenFile(dir, levelDBOptions)
if err != nil {
return nil, err
}
return &levelDBStore{
db: db,
inbox: make(chan func(), 16),
clock: defaultClock{},
}, nil
}
func newMemoryLevelDBStore() (*levelDBStore, error) {
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
return nil, err
}
return &levelDBStore{
db: db,
inbox: make(chan func(), 16),
clock: defaultClock{},
}, nil
}
func (s *levelDBStore) put(key string, rec DatabaseRecord) error {
t0 := time.Now()
defer func() {
databaseOperationSeconds.WithLabelValues(dbOpPut).Observe(time.Since(t0).Seconds())
}()
rc := make(chan error)
s.inbox <- func() {
size := rec.Size()
if len(s.marshalBuf) < size {
s.marshalBuf = make([]byte, size)
}
n, _ := rec.MarshalTo(s.marshalBuf)
rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
}
err := <-rc
if err != nil {
databaseOperations.WithLabelValues(dbOpPut, dbResError).Inc()
} else {
databaseOperations.WithLabelValues(dbOpPut, dbResSuccess).Inc()
}
return err
}
func (s *levelDBStore) merge(key string, addrs []DatabaseAddress, seen int64) error {
t0 := time.Now()
defer func() {
databaseOperationSeconds.WithLabelValues(dbOpMerge).Observe(time.Since(t0).Seconds())
}()
rc := make(chan error)
newRec := DatabaseRecord{
Addresses: addrs,
Seen: seen,
}
s.inbox <- func() {
// grab the existing record
oldRec, err := s.get(key)
if err != nil {
// "not found" is not an error from get, so this is serious
// stuff only
rc <- err
return
}
newRec = merge(newRec, oldRec)
// We replicate s.put() functionality here ourselves instead of
// calling it because we want to serialize our get above together
// with the put in the same function.
size := newRec.Size()
if len(s.marshalBuf) < size {
s.marshalBuf = make([]byte, size)
}
n, _ := newRec.MarshalTo(s.marshalBuf)
rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
}
err := <-rc
if err != nil {
databaseOperations.WithLabelValues(dbOpMerge, dbResError).Inc()
} else {
databaseOperations.WithLabelValues(dbOpMerge, dbResSuccess).Inc()
}
return err
}
func (s *levelDBStore) get(key string) (DatabaseRecord, error) {
t0 := time.Now()
defer func() {
databaseOperationSeconds.WithLabelValues(dbOpGet).Observe(time.Since(t0).Seconds())
}()
keyBs := []byte(key)
val, err := s.db.Get(keyBs, nil)
if err == leveldb.ErrNotFound {
databaseOperations.WithLabelValues(dbOpGet, dbResNotFound).Inc()
return DatabaseRecord{}, nil
}
if err != nil {
databaseOperations.WithLabelValues(dbOpGet, dbResError).Inc()
return DatabaseRecord{}, err
}
var rec DatabaseRecord
if err := rec.Unmarshal(val); err != nil {
databaseOperations.WithLabelValues(dbOpGet, dbResUnmarshalError).Inc()
return DatabaseRecord{}, nil
}
rec.Addresses = expire(rec.Addresses, s.clock.Now().UnixNano())
databaseOperations.WithLabelValues(dbOpGet, dbResSuccess).Inc()
return rec, nil
}
func (s *levelDBStore) Serve(ctx context.Context) error {
t := time.NewTimer(0)
defer t.Stop()
defer s.db.Close()
// Start the statistics serve routine. It will exit with us when
// statisticsTrigger is closed.
statisticsTrigger := make(chan struct{})
statisticsDone := make(chan struct{})
go s.statisticsServe(statisticsTrigger, statisticsDone)
loop:
for {
select {
case fn := <-s.inbox:
// Run function in serialized order.
fn()
case <-t.C:
// Trigger the statistics routine to do its thing in the
// background.
statisticsTrigger <- struct{}{}
case <-statisticsDone:
// The statistics routine is done with one iteratation, schedule
// the next.
t.Reset(databaseStatisticsInterval)
case <-ctx.Done():
// We're done.
close(statisticsTrigger)
break loop
}
}
// Also wait for statisticsServe to return
<-statisticsDone
return nil
}
func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
defer close(done)
for range trigger {
t0 := time.Now()
nowNanos := t0.UnixNano()
cutoff24h := t0.Add(-24 * time.Hour).UnixNano()
cutoff1w := t0.Add(-7 * 24 * time.Hour).UnixNano()
cutoff2Mon := t0.Add(-60 * 24 * time.Hour).UnixNano()
current, currentIPv4, currentIPv6, last24h, last1w, inactive, errors := 0, 0, 0, 0, 0, 0, 0
iter := s.db.NewIterator(&util.Range{}, nil)
for iter.Next() {
// Attempt to unmarshal the record and count the
// failure if there's something wrong with it.
var rec DatabaseRecord
if err := rec.Unmarshal(iter.Value()); err != nil {
errors++
continue
}
// If there are addresses that have not expired it's a current
// record, otherwise account it based on when it was last seen
// (last 24 hours or last week) or finally as inactice.
addrs := expire(rec.Addresses, nowNanos)
switch {
case len(addrs) > 0:
current++
seenIPv4, seenIPv6 := false, false
for _, addr := range addrs {
uri, err := url.Parse(addr.Address)
if err != nil {
continue
}
host, _, err := net.SplitHostPort(uri.Host)
if err != nil {
continue
}
if ip := net.ParseIP(host); ip != nil && ip.To4() != nil {
seenIPv4 = true
} else if ip != nil {
seenIPv6 = true
}
if seenIPv4 && seenIPv6 {
break
}
}
if seenIPv4 {
currentIPv4++
}
if seenIPv6 {
currentIPv6++
}
case rec.Seen > cutoff24h:
last24h++
case rec.Seen > cutoff1w:
last1w++
case rec.Seen > cutoff2Mon:
inactive++
case rec.Missed < cutoff2Mon:
// It hasn't been seen lately and we haven't recorded
// someone asking for this device in a long time either;
// delete the record.
if err := s.db.Delete(iter.Key(), nil); err != nil {
databaseOperations.WithLabelValues(dbOpDelete, dbResError).Inc()
} else {
databaseOperations.WithLabelValues(dbOpDelete, dbResSuccess).Inc()
}
default:
inactive++
}
}
iter.Release()
databaseKeys.WithLabelValues("current").Set(float64(current))
databaseKeys.WithLabelValues("currentIPv4").Set(float64(currentIPv4))
databaseKeys.WithLabelValues("currentIPv6").Set(float64(currentIPv6))
databaseKeys.WithLabelValues("last24h").Set(float64(last24h))
databaseKeys.WithLabelValues("last1w").Set(float64(last1w))
databaseKeys.WithLabelValues("inactive").Set(float64(inactive))
databaseKeys.WithLabelValues("error").Set(float64(errors))
databaseStatisticsSeconds.Set(time.Since(t0).Seconds())
// Signal that we are done and can be scheduled again.
done <- struct{}{}
}
}
// merge returns the merged result of the two database records a and b. The
// result is the union of the two address sets, with the newer expiry time
// chosen for any duplicates.
func merge(a, b DatabaseRecord) DatabaseRecord {
// Both lists must be sorted for this to work.
if !sort.IsSorted(databaseAddressOrder(a.Addresses)) {
log.Println("Warning: bug: addresses not correctly sorted in merge")
a.Addresses = sortedAddressCopy(a.Addresses)
}
if !sort.IsSorted(databaseAddressOrder(b.Addresses)) {
// no warning because this is the side we read from disk and it may
// legitimately predate correct sorting.
b.Addresses = sortedAddressCopy(b.Addresses)
}
res := DatabaseRecord{
Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
Seen: a.Seen,
}
if b.Seen > a.Seen {
res.Seen = b.Seen
}
aIdx := 0
bIdx := 0
aAddrs := a.Addresses
bAddrs := b.Addresses
loop:
for {
switch {
case aIdx == len(aAddrs) && bIdx == len(bAddrs):
// both lists are exhausted, we are done
break loop
case aIdx == len(aAddrs):
// a is exhausted, pick from b and continue
res.Addresses = append(res.Addresses, bAddrs[bIdx])
bIdx++
continue
case bIdx == len(bAddrs):
// b is exhausted, pick from a and continue
res.Addresses = append(res.Addresses, aAddrs[aIdx])
aIdx++
continue
}
// We have values left on both sides.
aVal := aAddrs[aIdx]
bVal := bAddrs[bIdx]
switch {
case aVal.Address == bVal.Address:
// update for same address, pick newer
if aVal.Expires > bVal.Expires {
res.Addresses = append(res.Addresses, aVal)
} else {
res.Addresses = append(res.Addresses, bVal)
}
aIdx++
bIdx++
case aVal.Address < bVal.Address:
// a is smallest, pick it and continue
res.Addresses = append(res.Addresses, aVal)
aIdx++
default:
// b is smallest, pick it and continue
res.Addresses = append(res.Addresses, bVal)
bIdx++
}
}
return res
}
// expire returns the list of addresses after removing expired entries.
// Expiration happen in place, so the slice given as the parameter is
// destroyed. Internal order is not preserved.
func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
i := 0
for i < len(addrs) {
if addrs[i].Expires < now {
addrs = sliceutil.RemoveAndZero(addrs, i)
continue
}
i++
}
return addrs
}
func sortedAddressCopy(addrs []DatabaseAddress) []DatabaseAddress {
sorted := make([]DatabaseAddress, len(addrs))
copy(sorted, addrs)
sort.Sort(databaseAddressOrder(sorted))
return sorted
}
type databaseAddressOrder []DatabaseAddress
func (s databaseAddressOrder) Less(a, b int) bool {
return s[a].Address < s[b].Address
}
func (s databaseAddressOrder) Swap(a, b int) {
s[a], s[b] = s[b], s[a]
}
func (s databaseAddressOrder) Len() int {
return len(s)
}

View File

@@ -1,847 +0,0 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: database.proto
package main
import (
fmt "fmt"
_ "github.com/gogo/protobuf/gogoproto"
proto "github.com/gogo/protobuf/proto"
io "io"
math "math"
math_bits "math/bits"
)
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
type DatabaseRecord struct {
Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
Misses int32 `protobuf:"varint,2,opt,name=misses,proto3" json:"misses,omitempty"`
Seen int64 `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
Missed int64 `protobuf:"varint,4,opt,name=missed,proto3" json:"missed,omitempty"`
}
func (m *DatabaseRecord) Reset() { *m = DatabaseRecord{} }
func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
func (*DatabaseRecord) ProtoMessage() {}
func (*DatabaseRecord) Descriptor() ([]byte, []int) {
return fileDescriptor_b90fe3356ea5df07, []int{0}
}
func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
xxx_messageInfo_DatabaseRecord.Merge(m, src)
}
func (m *DatabaseRecord) XXX_Size() int {
return m.Size()
}
func (m *DatabaseRecord) XXX_DiscardUnknown() {
xxx_messageInfo_DatabaseRecord.DiscardUnknown(m)
}
var xxx_messageInfo_DatabaseRecord proto.InternalMessageInfo
type ReplicationRecord struct {
Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses"`
Seen int64 `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
}
func (m *ReplicationRecord) Reset() { *m = ReplicationRecord{} }
func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
func (*ReplicationRecord) ProtoMessage() {}
func (*ReplicationRecord) Descriptor() ([]byte, []int) {
return fileDescriptor_b90fe3356ea5df07, []int{1}
}
func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
xxx_messageInfo_ReplicationRecord.Merge(m, src)
}
func (m *ReplicationRecord) XXX_Size() int {
return m.Size()
}
func (m *ReplicationRecord) XXX_DiscardUnknown() {
xxx_messageInfo_ReplicationRecord.DiscardUnknown(m)
}
var xxx_messageInfo_ReplicationRecord proto.InternalMessageInfo
type DatabaseAddress struct {
Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
Expires int64 `protobuf:"varint,2,opt,name=expires,proto3" json:"expires,omitempty"`
}
func (m *DatabaseAddress) Reset() { *m = DatabaseAddress{} }
func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
func (*DatabaseAddress) ProtoMessage() {}
func (*DatabaseAddress) Descriptor() ([]byte, []int) {
return fileDescriptor_b90fe3356ea5df07, []int{2}
}
func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
xxx_messageInfo_DatabaseAddress.Merge(m, src)
}
func (m *DatabaseAddress) XXX_Size() int {
return m.Size()
}
func (m *DatabaseAddress) XXX_DiscardUnknown() {
xxx_messageInfo_DatabaseAddress.DiscardUnknown(m)
}
var xxx_messageInfo_DatabaseAddress proto.InternalMessageInfo
func init() {
proto.RegisterType((*DatabaseRecord)(nil), "main.DatabaseRecord")
proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
}
func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
var fileDescriptor_b90fe3356ea5df07 = []byte{
// 270 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
}
func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.Missed != 0 {
i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
i--
dAtA[i] = 0x20
}
if m.Seen != 0 {
i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
i--
dAtA[i] = 0x18
}
if m.Misses != 0 {
i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
i--
dAtA[i] = 0x10
}
if len(m.Addresses) > 0 {
for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
{
size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintDatabase(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0xa
}
}
return len(dAtA) - i, nil
}
func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.Seen != 0 {
i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
i--
dAtA[i] = 0x18
}
if len(m.Addresses) > 0 {
for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
{
size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintDatabase(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0x12
}
}
if len(m.Key) > 0 {
i -= len(m.Key)
copy(dAtA[i:], m.Key)
i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
i--
dAtA[i] = 0xa
}
return len(dAtA) - i, nil
}
func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.Expires != 0 {
i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
i--
dAtA[i] = 0x10
}
if len(m.Address) > 0 {
i -= len(m.Address)
copy(dAtA[i:], m.Address)
i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
i--
dAtA[i] = 0xa
}
return len(dAtA) - i, nil
}
func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
offset -= sovDatabase(v)
base := offset
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
dAtA[offset] = uint8(v)
return base
}
func (m *DatabaseRecord) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
if len(m.Addresses) > 0 {
for _, e := range m.Addresses {
l = e.Size()
n += 1 + l + sovDatabase(uint64(l))
}
}
if m.Misses != 0 {
n += 1 + sovDatabase(uint64(m.Misses))
}
if m.Seen != 0 {
n += 1 + sovDatabase(uint64(m.Seen))
}
if m.Missed != 0 {
n += 1 + sovDatabase(uint64(m.Missed))
}
return n
}
func (m *ReplicationRecord) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.Key)
if l > 0 {
n += 1 + l + sovDatabase(uint64(l))
}
if len(m.Addresses) > 0 {
for _, e := range m.Addresses {
l = e.Size()
n += 1 + l + sovDatabase(uint64(l))
}
}
if m.Seen != 0 {
n += 1 + sovDatabase(uint64(m.Seen))
}
return n
}
func (m *DatabaseAddress) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.Address)
if l > 0 {
n += 1 + l + sovDatabase(uint64(l))
}
if m.Expires != 0 {
n += 1 + sovDatabase(uint64(m.Expires))
}
return n
}
func sovDatabase(x uint64) (n int) {
return (math_bits.Len64(x|1) + 6) / 7
}
func sozDatabase(x uint64) (n int) {
return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: DatabaseRecord: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: DatabaseRecord: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthDatabase
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthDatabase
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Addresses = append(m.Addresses, DatabaseAddress{})
if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Misses", wireType)
}
m.Misses = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Misses |= int32(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
}
m.Seen = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Seen |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Missed", wireType)
}
m.Missed = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Missed |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipDatabase(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthDatabase
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: ReplicationRecord: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: ReplicationRecord: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthDatabase
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthDatabase
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Key = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthDatabase
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthDatabase
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Addresses = append(m.Addresses, DatabaseAddress{})
if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
}
m.Seen = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Seen |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipDatabase(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthDatabase
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: DatabaseAddress: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: DatabaseAddress: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Address", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthDatabase
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthDatabase
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Address = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Expires", wireType)
}
m.Expires = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Expires |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipDatabase(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthDatabase
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipDatabase(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
depth := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDatabase
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDatabase
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if dAtA[iNdEx-1] < 0x80 {
break
}
}
case 1:
iNdEx += 8
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDatabase
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if length < 0 {
return 0, ErrInvalidLengthDatabase
}
iNdEx += length
case 3:
depth++
case 4:
if depth == 0 {
return 0, ErrUnexpectedEndOfGroupDatabase
}
depth--
case 5:
iNdEx += 4
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
if iNdEx < 0 {
return 0, ErrInvalidLengthDatabase
}
if depth == 0 {
return iNdEx, nil
}
}
return 0, io.ErrUnexpectedEOF
}
var (
ErrInvalidLengthDatabase = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowDatabase = fmt.Errorf("proto: integer overflow")
ErrUnexpectedEndOfGroupDatabase = fmt.Errorf("proto: unexpected end of group")
)

View File

@@ -1,36 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
syntax = "proto3";
package main;
import "repos/protobuf/gogoproto/gogo.proto";
option (gogoproto.goproto_getters_all) = false;
option (gogoproto.goproto_unkeyed_all) = false;
option (gogoproto.goproto_unrecognized_all) = false;
option (gogoproto.goproto_sizecache_all) = false;
message DatabaseRecord {
repeated DatabaseAddress addresses = 1 [(gogoproto.nullable) = false];
int32 misses = 2; // Number of lookups* without hits
int64 seen = 3; // Unix nanos, last device announce
int64 missed = 4; // Unix nanos, last* failed lookup
}
// *) Not every lookup results in a write, so may not be completely accurate
message ReplicationRecord {
string key = 1;
repeated DatabaseAddress addresses = 2 [(gogoproto.nullable) = false];
int64 seen = 3; // Unix nanos, last device announce
}
message DatabaseAddress {
string address = 1;
int64 expires = 2; // Unix nanos
}

View File

@@ -1,211 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"context"
"fmt"
"testing"
"time"
)
func TestDatabaseGetSet(t *testing.T) {
db, err := newMemoryLevelDBStore()
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
go db.Serve(ctx)
defer cancel()
// Check missing record
rec, err := db.get("abcd")
if err != nil {
t.Error("not found should not be an error")
}
if len(rec.Addresses) != 0 {
t.Error("addresses should be empty")
}
if rec.Misses != 0 {
t.Error("missing should be zero")
}
// Set up a clock
now := time.Now()
tc := &testClock{now}
db.clock = tc
// Put a record
rec.Addresses = []DatabaseAddress{
{Address: "tcp://1.2.3.4:5", Expires: tc.Now().Add(time.Minute).UnixNano()},
}
if err := db.put("abcd", rec); err != nil {
t.Fatal(err)
}
// Verify it
rec, err = db.get("abcd")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 1 {
t.Log(rec.Addresses)
t.Fatal("should have one address")
}
if rec.Addresses[0].Address != "tcp://1.2.3.4:5" {
t.Log(rec.Addresses)
t.Error("incorrect address")
}
// Wind the clock one half expiry, and merge in a new address
tc.wind(30 * time.Second)
addrs := []DatabaseAddress{
{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
}
if err := db.merge("abcd", addrs, tc.Now().UnixNano()); err != nil {
t.Fatal(err)
}
// Verify it
rec, err = db.get("abcd")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 2 {
t.Log(rec.Addresses)
t.Fatal("should have two addresses")
}
if rec.Addresses[0].Address != "tcp://1.2.3.4:5" {
t.Log(rec.Addresses)
t.Error("incorrect address[0]")
}
if rec.Addresses[1].Address != "tcp://6.7.8.9:0" {
t.Log(rec.Addresses)
t.Error("incorrect address[1]")
}
// Pass the first expiry time
tc.wind(45 * time.Second)
// Verify it
rec, err = db.get("abcd")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 1 {
t.Log(rec.Addresses)
t.Fatal("should have one address")
}
if rec.Addresses[0].Address != "tcp://6.7.8.9:0" {
t.Log(rec.Addresses)
t.Error("incorrect address")
}
// Put a record with misses
rec = DatabaseRecord{Misses: 42, Missed: tc.Now().UnixNano()}
if err := db.put("efgh", rec); err != nil {
t.Fatal(err)
}
// Verify it
rec, err = db.get("efgh")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 0 {
t.Log(rec.Addresses)
t.Fatal("should have no addresses")
}
if rec.Misses != 42 {
t.Log(rec.Misses)
t.Error("incorrect misses")
}
// Set an address
addrs = []DatabaseAddress{
{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
}
if err := db.merge("efgh", addrs, tc.Now().UnixNano()); err != nil {
t.Fatal(err)
}
// Verify it
rec, err = db.get("efgh")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 1 {
t.Log(rec.Addresses)
t.Fatal("should have one address")
}
if rec.Misses != 0 {
t.Log(rec.Misses)
t.Error("should have no misses")
}
}
func TestFilter(t *testing.T) {
// all cases are expired with t=10
cases := []struct {
a []DatabaseAddress
b []DatabaseAddress
}{
{
a: nil,
b: nil,
},
{
a: []DatabaseAddress{{Address: "a", Expires: 9}, {Address: "b", Expires: 9}, {Address: "c", Expires: 9}},
b: []DatabaseAddress{},
},
{
a: []DatabaseAddress{{Address: "a", Expires: 10}},
b: []DatabaseAddress{{Address: "a", Expires: 10}},
},
{
a: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
b: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
},
{
a: []DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
b: []DatabaseAddress{{Address: "b", Expires: 15}, {Address: "d", Expires: 15}},
},
}
for _, tc := range cases {
res := expire(tc.a, 10)
if fmt.Sprint(res) != fmt.Sprint(tc.b) {
t.Errorf("Incorrect result %v, expected %v", res, tc.b)
}
}
}
type testClock struct {
now time.Time
}
func (t *testClock) wind(d time.Duration) {
t.now = t.now.Add(d)
}
func (t *testClock) Now() time.Time {
t.now = t.now.Add(time.Nanosecond)
return t.now
}

32
cmd/stdiscosrv/db.go Normal file
View File

@@ -0,0 +1,32 @@
// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
package main
import (
"database/sql"
"fmt"
)
type setupFunc func(db *sql.DB) error
type compileFunc func(db *sql.DB) (map[string]*sql.Stmt, error)
var (
setupFuncs = make(map[string]setupFunc)
compileFuncs = make(map[string]compileFunc)
)
func register(name string, setup setupFunc, compile compileFunc) {
setupFuncs[name] = setup
compileFuncs[name] = compile
}
func setup(backend string, db *sql.DB) (map[string]*sql.Stmt, error) {
setup, ok := setupFuncs[backend]
if !ok {
return nil, fmt.Errorf("Unsupported backend")
}
if err := setup(db); err != nil {
return nil, err
}
return compileFuncs[backend](db)
}

View File

@@ -1,4 +0,0 @@
[stdiscosrv]
title=Syncthing discovery server
description=Lets syncthing clients discover each other
ports=8443/tcp

View File

@@ -1,3 +0,0 @@
# Default settings for syncthing-discosrv (stdiscosrv).
## Add Options here:
DISCOSRV_OPTS=

View File

@@ -1,25 +0,0 @@
[Unit]
Description=Syncthing Discovery Server
After=network.target
Documentation=man:stdiscosrv(1)
[Service]
WorkingDirectory=/var/lib/syncthing-discosrv
EnvironmentFile=/etc/default/syncthing-discosrv
ExecStart=/usr/bin/stdiscosrv $DISCOSRV_OPTS
# Hardening
User=syncthing-discosrv
Group=syncthing
ProtectSystem=strict
ReadWritePaths=/var/lib/syncthing-discosrv
NoNewPrivileges=true
PrivateTmp=true
PrivateDevices=true
ProtectHome=true
SystemCallArchitectures=native
MemoryDenyWriteExecute=true
[Install]
WantedBy=multi-user.target
Alias=syncthing-discosrv.service

View File

@@ -1,221 +1,146 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
package main
import (
"context"
"crypto/tls"
"database/sql"
"flag"
"fmt"
"log"
"net"
"net/http"
"os"
"runtime"
"strings"
"strconv"
"time"
"github.com/prometheus/client_golang/prometheus/promhttp"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/tlsutil"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/thejerf/suture/v4"
"github.com/thejerf/suture"
)
const (
addressExpiryTime = 2 * time.Hour
databaseStatisticsInterval = 5 * time.Minute
// Reannounce-After is set to reannounceAfterSeconds +
// random(reannounzeFuzzSeconds), similar for Retry-After
reannounceAfterSeconds = 3300
reannounzeFuzzSeconds = 300
errorRetryAfterSeconds = 1500
errorRetryFuzzSeconds = 300
// Retry for not found is minSeconds + failures * incSeconds +
// random(fuzz), where failures is the number of consecutive lookups
// with no answer, up to maxSeconds. The fuzz is applied after capping
// to maxSeconds.
notFoundRetryMinSeconds = 60
notFoundRetryMaxSeconds = 3540
notFoundRetryIncSeconds = 10
notFoundRetryFuzzSeconds = 60
// How often (in requests) we serialize the missed counter to database.
notFoundMissesWriteInterval = 10
httpReadTimeout = 5 * time.Second
httpWriteTimeout = 5 * time.Second
httpMaxHeaderBytes = 1 << 10
// Size of the replication outbox channel
replicationOutboxSize = 10000
minNegCache = 60 // seconds
maxNegCache = 3600 // seconds
maxDeviceAge = 7 * 86400 // one week, in seconds
)
// These options make the database a little more optimized for writes, at
// the expense of some memory usage and risk of losing writes in a (system)
// crash.
var levelDBOptions = &opt.Options{
NoSync: true,
WriteBuffer: 32 << 20, // default 4<<20
var (
Version string
BuildStamp string
BuildUser string
BuildHost string
BuildDate time.Time
LongVersion string
)
func init() {
stamp, _ := strconv.Atoi(BuildStamp)
BuildDate = time.Unix(int64(stamp), 0)
date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
LongVersion = fmt.Sprintf(`stdiscosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
}
var debug = false
var (
lruSize = 10240
limitAvg = 5
limitBurst = 20
globalStats stats
statsFile string
backend = "ql"
dsn = getEnvDefault("STDISCOSRV_DB_DSN", "memory://stdiscosrv")
certFile = "cert.pem"
keyFile = "key.pem"
debug = false
useHTTP = false
)
func main() {
const (
cleanIntv = 1 * time.Hour
statsIntv = 5 * time.Minute
)
var listen string
var dir string
var metricsListen string
var replicationListen string
var replicationPeers string
var certFile string
var keyFile string
var replCertFile string
var replKeyFile string
var useHTTP bool
var largeDB bool
log.SetOutput(os.Stdout)
log.SetFlags(0)
flag.StringVar(&certFile, "cert", "./cert.pem", "Certificate file")
flag.StringVar(&keyFile, "key", "./key.pem", "Key file")
flag.StringVar(&dir, "db-dir", "./discovery.db", "Database directory")
flag.BoolVar(&debug, "debug", false, "Print debug output")
flag.BoolVar(&useHTTP, "http", false, "Listen on HTTP (behind an HTTPS proxy)")
flag.StringVar(&listen, "listen", ":8443", "Listen address")
flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
flag.StringVar(&replCertFile, "replication-cert", "", "Certificate file for replication")
flag.StringVar(&replKeyFile, "replication-key", "", "Key file for replication")
flag.BoolVar(&largeDB, "large-db", false, "Use larger database settings")
showVersion := flag.Bool("version", false, "Show version")
flag.IntVar(&lruSize, "limit-cache", lruSize, "Limiter cache entries")
flag.IntVar(&limitAvg, "limit-avg", limitAvg, "Allowed average package rate, per 10 s")
flag.IntVar(&limitBurst, "limit-burst", limitBurst, "Allowed burst size, packets")
flag.StringVar(&statsFile, "stats-file", statsFile, "File to write periodic operation stats to")
flag.StringVar(&backend, "db-backend", backend, "Database backend to use")
flag.StringVar(&dsn, "db-dsn", dsn, "Database DSN")
flag.StringVar(&certFile, "cert", certFile, "Certificate file")
flag.StringVar(&keyFile, "key", keyFile, "Key file")
flag.BoolVar(&debug, "debug", debug, "Debug")
flag.BoolVar(&useHTTP, "http", useHTTP, "Listen on HTTP (behind an HTTPS proxy)")
flag.Parse()
log.Println(build.LongVersionFor("stdiscosrv"))
if *showVersion {
return
}
log.Println(LongVersion)
buildInfo.WithLabelValues(build.Version, runtime.Version(), build.User, build.Date.UTC().Format("2006-01-02T15:04:05Z")).Set(1)
if largeDB {
levelDBOptions.BlockCacheCapacity = 64 << 20
levelDBOptions.BlockSize = 64 << 10
levelDBOptions.CompactionTableSize = 16 << 20
levelDBOptions.CompactionTableSizeMultiplier = 2.0
levelDBOptions.WriteBuffer = 64 << 20
levelDBOptions.CompactionL0Trigger = 8
}
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if os.IsNotExist(err) {
log.Println("Failed to load keypair. Generating one, this might take a while...")
cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 20*365)
var cert tls.Certificate
var err error
if !useHTTP {
cert, err = tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
log.Fatalln("Failed to generate X509 key pair:", err)
}
} else if err != nil {
log.Fatalln("Failed to load keypair:", err)
}
devID := protocol.NewDeviceID(cert.Certificate[0])
log.Println("Server device ID is", devID)
replCert := cert
if replCertFile != "" && replKeyFile != "" {
replCert, err = tls.LoadX509KeyPair(replCertFile, replKeyFile)
if err != nil {
log.Fatalln("Failed to load replication keypair:", err)
}
}
replDevID := protocol.NewDeviceID(replCert.Certificate[0])
log.Println("Replication device ID is", replDevID)
// Parse the replication specs, if any.
var allowedReplicationPeers []protocol.DeviceID
var replicationDestinations []string
parts := strings.Split(replicationPeers, ",")
for _, part := range parts {
if part == "" {
continue
}
fields := strings.Split(part, "@")
switch len(fields) {
case 2:
// This is an id@address specification. Grab the address for the
// destination list. Try to resolve it once to catch obvious
// syntax errors here rather than having the sender service fail
// repeatedly later.
_, err := net.ResolveTCPAddr("tcp", fields[1])
log.Println("Failed to load keypair. Generating one, this might take a while...")
cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 3072)
if err != nil {
log.Fatalln("Resolving address:", err)
log.Fatalln("Failed to generate X509 key pair:", err)
}
replicationDestinations = append(replicationDestinations, fields[1])
fallthrough // N.B.
case 1:
// The first part is always a device ID.
id, err := protocol.DeviceIDFromString(fields[0])
if err != nil {
log.Fatalln("Parsing device ID:", err)
}
if id == protocol.EmptyDeviceID {
log.Fatalf("Missing device ID for peer in %q", part)
}
allowedReplicationPeers = append(allowedReplicationPeers, id)
default:
log.Fatalln("Unrecognized replication spec:", part)
}
devID := protocol.NewDeviceID(cert.Certificate[0])
log.Println("Server device ID is", devID)
}
// Root of the service tree.
main := suture.New("main", suture.Spec{
PassThroughPanics: true,
db, err := sql.Open(backend, dsn)
if err != nil {
log.Fatalln("sql.Open:", err)
}
prep, err := setup(backend, db)
if err != nil {
log.Fatalln("Setup:", err)
}
main := suture.NewSimple("main")
main.Add(&querysrv{
addr: listen,
cert: cert,
db: db,
prep: prep,
})
// Start the database.
db, err := newLevelDBStore(dir)
if err != nil {
log.Fatalln("Open database:", err)
}
main.Add(db)
main.Add(&cleansrv{
intv: cleanIntv,
db: db,
prep: prep,
})
// Start any replication senders.
var repl replicationMultiplexer
for _, dst := range replicationDestinations {
rs := newReplicationSender(dst, replCert, allowedReplicationPeers)
main.Add(rs)
repl = append(repl, rs)
}
main.Add(&statssrv{
intv: statsIntv,
file: statsFile,
db: db,
})
// If we have replication configured, start the replication listener.
if len(allowedReplicationPeers) > 0 {
rl := newReplicationListener(replicationListen, replCert, allowedReplicationPeers, db)
main.Add(rl)
}
// Start the main API server.
qs := newAPISrv(listen, cert, db, repl, useHTTP)
main.Add(qs)
// If we have a metrics port configured, start a metrics handler.
if metricsListen != "" {
go func() {
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
log.Fatal(http.ListenAndServe(metricsListen, mux))
}()
}
// Engage!
main.Serve(context.Background())
globalStats.Reset()
main.Serve()
}
func getEnvDefault(key, def string) string {
if val := os.Getenv(key); val != "" {
return val
}
return def
}
func next(intv time.Duration) time.Duration {
t0 := time.Now()
t1 := t0.Add(intv).Truncate(intv)
return t1.Sub(t0)
}

98
cmd/stdiscosrv/psql.go Normal file
View File

@@ -0,0 +1,98 @@
// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
package main
import (
"database/sql"
"fmt"
_ "github.com/lib/pq"
)
func init() {
register("postgres", postgresSetup, postgresCompile)
}
func postgresSetup(db *sql.DB) error {
var err error
db.SetMaxIdleConns(4)
db.SetMaxOpenConns(8)
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Devices (
DeviceID CHAR(63) NOT NULL PRIMARY KEY,
Seen TIMESTAMP NOT NULL
)`)
if err != nil {
return err
}
var tmp string
row := db.QueryRow(`SELECT 'DevicesDeviceIDIndex'::regclass`)
if err = row.Scan(&tmp); err != nil {
_, err = db.Exec(`CREATE INDEX DevicesDeviceIDIndex ON Devices (DeviceID)`)
}
if err != nil {
return err
}
row = db.QueryRow(`SELECT 'DevicesSeenIndex'::regclass`)
if err = row.Scan(&tmp); err != nil {
_, err = db.Exec(`CREATE INDEX DevicesSeenIndex ON Devices (Seen)`)
}
if err != nil {
return err
}
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Addresses (
DeviceID CHAR(63) NOT NULL,
Seen TIMESTAMP NOT NULL,
Address VARCHAR(2048) NOT NULL
)`)
if err != nil {
return err
}
row = db.QueryRow(`SELECT 'AddressesDeviceIDSeenIndex'::regclass`)
if err = row.Scan(&tmp); err != nil {
_, err = db.Exec(`CREATE INDEX AddressesDeviceIDSeenIndex ON Addresses (DeviceID, Seen)`)
}
if err != nil {
return err
}
row = db.QueryRow(`SELECT 'AddressesDeviceIDAddressIndex'::regclass`)
if err = row.Scan(&tmp); err != nil {
_, err = db.Exec(`CREATE INDEX AddressesDeviceIDAddressIndex ON Addresses (DeviceID, Address)`)
}
if err != nil {
return err
}
return nil
}
func postgresCompile(db *sql.DB) (map[string]*sql.Stmt, error) {
stmts := map[string]string{
"cleanAddress": "DELETE FROM Addresses WHERE Seen < now() - '2 hour'::INTERVAL",
"cleanDevice": fmt.Sprintf("DELETE FROM Devices WHERE Seen < now() - '%d hour'::INTERVAL", maxDeviceAge/3600),
"countAddress": "SELECT count(*) FROM Addresses",
"countDevice": "SELECT count(*) FROM Devices",
"insertAddress": "INSERT INTO Addresses (DeviceID, Seen, Address) VALUES ($1, now(), $2)",
"insertDevice": "INSERT INTO Devices (DeviceID, Seen) VALUES ($1, now())",
"selectAddress": "SELECT Address FROM Addresses WHERE DeviceID=$1 AND Seen > now() - '1 hour'::INTERVAL ORDER BY random() LIMIT 16",
"selectDevice": "SELECT Seen FROM Devices WHERE DeviceID=$1",
"updateAddress": "UPDATE Addresses SET Seen=now() WHERE DeviceID=$1 AND Address=$2",
"updateDevice": "UPDATE Devices SET Seen=now() WHERE DeviceID=$1",
}
res := make(map[string]*sql.Stmt, len(stmts))
for key, stmt := range stmts {
prep, err := db.Prepare(stmt)
if err != nil {
return nil, err
}
res[key] = prep
}
return res, nil
}

81
cmd/stdiscosrv/ql.go Normal file
View File

@@ -0,0 +1,81 @@
// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
package main
import (
"database/sql"
"fmt"
"log"
"github.com/cznic/ql"
)
func init() {
ql.RegisterDriver()
register("ql", qlSetup, qlCompile)
}
func qlSetup(db *sql.DB) (err error) {
tx, err := db.Begin()
if err != nil {
return
}
defer func() {
if err == nil {
err = tx.Commit()
} else {
tx.Rollback()
}
}()
_, err = tx.Exec(`CREATE TABLE IF NOT EXISTS Devices (
DeviceID STRING NOT NULL,
Seen TIME NOT NULL
)`)
if err != nil {
return
}
if _, err = tx.Exec(`CREATE INDEX IF NOT EXISTS DevicesDeviceIDIndex ON Devices (DeviceID)`); err != nil {
return
}
_, err = tx.Exec(`CREATE TABLE IF NOT EXISTS Addresses (
DeviceID STRING NOT NULL,
Seen TIME NOT NULL,
Address STRING NOT NULL,
)`)
if err != nil {
return
}
_, err = tx.Exec(`CREATE INDEX IF NOT EXISTS AddressesDeviceIDAddressIndex ON Addresses (DeviceID, Address)`)
return
}
func qlCompile(db *sql.DB) (map[string]*sql.Stmt, error) {
stmts := map[string]string{
"cleanAddress": `DELETE FROM Addresses WHERE Seen < now() - duration("2h")`,
"cleanDevice": fmt.Sprintf(`DELETE FROM Devices WHERE Seen < now() - duration("%dh")`, maxDeviceAge/3600),
"countAddress": "SELECT count(*) FROM Addresses",
"countDevice": "SELECT count(*) FROM Devices",
"insertAddress": "INSERT INTO Addresses (DeviceID, Seen, Address) VALUES ($1, now(), $2)",
"insertDevice": "INSERT INTO Devices (DeviceID, Seen) VALUES ($1, now())",
"selectAddress": `SELECT Address from Addresses WHERE DeviceID==$1 AND Seen > now() - duration("1h") LIMIT 16`,
"selectDevice": "SELECT Seen FROM Devices WHERE DeviceID==$1",
"updateAddress": "UPDATE Addresses Seen=now() WHERE DeviceID==$1 AND Address==$2",
"updateDevice": "UPDATE Devices Seen=now() WHERE DeviceID==$1",
}
res := make(map[string]*sql.Stmt, len(stmts))
for key, stmt := range stmts {
prep, err := db.Prepare(stmt)
if err != nil {
log.Println("Failed to compile", stmt)
return nil, err
}
res[key] = prep
}
return res, nil
}

492
cmd/stdiscosrv/querysrv.go Normal file
View File

@@ -0,0 +1,492 @@
// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
package main
import (
"bytes"
"crypto/tls"
"database/sql"
"encoding/json"
"encoding/pem"
"fmt"
"log"
"math/rand"
"net"
"net/http"
"net/url"
"strconv"
"sync"
"time"
"github.com/golang/groupcache/lru"
"github.com/syncthing/syncthing/lib/protocol"
"golang.org/x/net/context"
"golang.org/x/time/rate"
)
type querysrv struct {
addr string
db *sql.DB
prep map[string]*sql.Stmt
limiter *safeCache
cert tls.Certificate
listener net.Listener
}
type announcement struct {
Seen time.Time `json:"seen"`
Addresses []string `json:"addresses"`
}
type safeCache struct {
*lru.Cache
mut sync.Mutex
}
func (s *safeCache) Get(key string) (val interface{}, ok bool) {
s.mut.Lock()
val, ok = s.Cache.Get(key)
s.mut.Unlock()
return
}
func (s *safeCache) Add(key string, val interface{}) {
s.mut.Lock()
s.Cache.Add(key, val)
s.mut.Unlock()
}
type requestID int64
func (i requestID) String() string {
return fmt.Sprintf("%016x", int64(i))
}
type contextKey int
const idKey contextKey = iota
func negCacheFor(lastSeen time.Time) int {
since := time.Since(lastSeen).Seconds()
if since >= maxDeviceAge {
return maxNegCache
}
if since < 0 {
// That's weird
return minNegCache
}
// Return a value linearly scaled from minNegCache (at zero seconds ago)
// to maxNegCache (at maxDeviceAge seconds ago).
r := since / maxDeviceAge
return int(minNegCache + r*(maxNegCache-minNegCache))
}
func (s *querysrv) Serve() {
s.limiter = &safeCache{
Cache: lru.New(lruSize),
}
if useHTTP {
listener, err := net.Listen("tcp", s.addr)
if err != nil {
log.Println("Listen:", err)
return
}
s.listener = listener
} else {
tlsCfg := &tls.Config{
Certificates: []tls.Certificate{s.cert},
ClientAuth: tls.RequestClientCert,
SessionTicketsDisabled: true,
MinVersion: tls.VersionTLS12,
CipherSuites: []uint16{
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
},
}
tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
if err != nil {
log.Println("Listen:", err)
return
}
s.listener = tlsListener
}
http.HandleFunc("/v2/", s.handler)
http.HandleFunc("/ping", handlePing)
srv := &http.Server{
ReadTimeout: 5 * time.Second,
WriteTimeout: 5 * time.Second,
MaxHeaderBytes: 1 << 10,
}
if err := srv.Serve(s.listener); err != nil {
log.Println("Serve:", err)
}
}
var topCtx = context.Background()
func (s *querysrv) handler(w http.ResponseWriter, req *http.Request) {
reqID := requestID(rand.Int63())
ctx := context.WithValue(topCtx, idKey, reqID)
if debug {
log.Println(reqID, req.Method, req.URL)
}
t0 := time.Now()
defer func() {
diff := time.Since(t0)
var comment string
if diff > time.Second {
comment = "(very slow request)"
} else if diff > 100*time.Millisecond {
comment = "(slow request)"
}
if comment != "" || debug {
log.Println(reqID, req.Method, req.URL, "completed in", diff, comment)
}
}()
var remoteIP net.IP
if useHTTP {
remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
} else {
addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
if err != nil {
log.Println("remoteAddr:", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
remoteIP = addr.IP
}
if s.limit(remoteIP) {
if debug {
log.Println(remoteIP, "is limited")
}
w.Header().Set("Retry-After", "60")
http.Error(w, "Too Many Requests", 429)
return
}
switch req.Method {
case "GET":
s.handleGET(ctx, w, req)
case "POST":
s.handlePOST(ctx, remoteIP, w, req)
default:
globalStats.Error()
http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
}
}
func (s *querysrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
reqID := ctx.Value(idKey).(requestID)
deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
if err != nil {
if debug {
log.Println(reqID, "bad device param")
}
globalStats.Error()
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
var ann announcement
ann.Seen, err = s.getDeviceSeen(deviceID)
negCache := strconv.Itoa(negCacheFor(ann.Seen))
w.Header().Set("Retry-After", negCache)
w.Header().Set("Cache-Control", "public, max-age="+negCache)
if err != nil {
// The device is not in the database.
globalStats.Query()
http.Error(w, "Not Found", http.StatusNotFound)
return
}
t0 := time.Now()
ann.Addresses, err = s.getAddresses(ctx, deviceID)
if err != nil {
log.Println(reqID, "getAddresses:", err)
globalStats.Error()
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
if debug {
log.Println(reqID, "getAddresses in", time.Since(t0))
}
globalStats.Query()
if len(ann.Addresses) == 0 {
http.Error(w, "Not Found", http.StatusNotFound)
return
}
globalStats.Answer()
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(ann)
}
func (s *querysrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
reqID := ctx.Value(idKey).(requestID)
rawCert := certificateBytes(req)
if rawCert == nil {
if debug {
log.Println(reqID, "no certificates")
}
globalStats.Error()
http.Error(w, "Forbidden", http.StatusForbidden)
return
}
var ann announcement
if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
if debug {
log.Println(reqID, "decode:", err)
}
globalStats.Error()
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
deviceID := protocol.NewDeviceID(rawCert)
// handleAnnounce returns *two* errors. The first indicates a problem with
// something the client posted to us. We should return a 400 Bad Request
// and not worry about it. The second indicates that the request was fine,
// but something internal messed up. We should log it and respond with a
// more apologetic 500 Internal Server Error.
userErr, internalErr := s.handleAnnounce(ctx, remoteIP, deviceID, ann.Addresses)
if userErr != nil {
if debug {
log.Println(reqID, "handleAnnounce:", userErr)
}
globalStats.Error()
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
if internalErr != nil {
log.Println(reqID, "handleAnnounce:", internalErr)
globalStats.Error()
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
globalStats.Announce()
// TODO: Slowly increase this for stable clients
w.Header().Set("Reannounce-After", "1800")
// We could return the lookup result here, but it's kind of unnecessarily
// expensive to go query the database again so we let the client decide to
// do a lookup if they really care.
w.WriteHeader(http.StatusNoContent)
}
func (s *querysrv) Stop() {
s.listener.Close()
}
func (s *querysrv) handleAnnounce(ctx context.Context, remote net.IP, deviceID protocol.DeviceID, addresses []string) (userErr, internalErr error) {
reqID := ctx.Value(idKey).(requestID)
tx, err := s.db.Begin()
if err != nil {
internalErr = err
return
}
defer func() {
// Since we return from a bunch of different places, we handle
// rollback in the defer.
if internalErr != nil || userErr != nil {
tx.Rollback()
}
}()
for _, annAddr := range addresses {
uri, err := url.Parse(annAddr)
if err != nil {
userErr = err
return
}
host, port, err := net.SplitHostPort(uri.Host)
if err != nil {
userErr = err
return
}
ip := net.ParseIP(host)
if host == "" || ip.IsUnspecified() {
// Do not use IPv6 remote address if requested scheme is tcp4
if uri.Scheme == "tcp4" && remote.To4() == nil {
continue
}
// Do not use IPv4 remote address if requested scheme is tcp6
if uri.Scheme == "tcp6" && remote.To4() != nil {
continue
}
host = remote.String()
}
uri.Host = net.JoinHostPort(host, port)
if err := s.updateAddress(ctx, tx, deviceID, uri.String()); err != nil {
internalErr = err
return
}
}
if err := s.updateDevice(ctx, tx, deviceID); err != nil {
internalErr = err
return
}
t0 := time.Now()
internalErr = tx.Commit()
if debug {
log.Println(reqID, "commit in", time.Since(t0))
}
return
}
func (s *querysrv) limit(remote net.IP) bool {
key := remote.String()
bkt, ok := s.limiter.Get(key)
if ok {
bkt := bkt.(*rate.Limiter)
if !bkt.Allow() {
// Rate limit exceeded; ignore packet
return true
}
} else {
// limitAvg is in packets per ten seconds.
s.limiter.Add(key, rate.NewLimiter(rate.Limit(limitAvg)/10, limitBurst))
}
return false
}
func (s *querysrv) updateDevice(ctx context.Context, tx *sql.Tx, device protocol.DeviceID) error {
reqID := ctx.Value(idKey).(requestID)
t0 := time.Now()
res, err := tx.Stmt(s.prep["updateDevice"]).Exec(device.String())
if err != nil {
return err
}
if debug {
log.Println(reqID, "updateDevice in", time.Since(t0))
}
if rows, _ := res.RowsAffected(); rows == 0 {
t0 = time.Now()
_, err := tx.Stmt(s.prep["insertDevice"]).Exec(device.String())
if err != nil {
return err
}
if debug {
log.Println(reqID, "insertDevice in", time.Since(t0))
}
}
return nil
}
func (s *querysrv) updateAddress(ctx context.Context, tx *sql.Tx, device protocol.DeviceID, uri string) error {
res, err := tx.Stmt(s.prep["updateAddress"]).Exec(device.String(), uri)
if err != nil {
return err
}
if rows, _ := res.RowsAffected(); rows == 0 {
_, err := tx.Stmt(s.prep["insertAddress"]).Exec(device.String(), uri)
if err != nil {
return err
}
}
return nil
}
func (s *querysrv) getAddresses(ctx context.Context, device protocol.DeviceID) ([]string, error) {
rows, err := s.prep["selectAddress"].Query(device.String())
if err != nil {
return nil, err
}
defer rows.Close()
var res []string
for rows.Next() {
var addr string
err := rows.Scan(&addr)
if err != nil {
log.Println("Scan:", err)
continue
}
res = append(res, addr)
}
return res, nil
}
func (s *querysrv) getDeviceSeen(device protocol.DeviceID) (time.Time, error) {
row := s.prep["selectDevice"].QueryRow(device.String())
var seen time.Time
if err := row.Scan(&seen); err != nil {
return time.Time{}, err
}
return seen.In(time.UTC), nil
}
func handlePing(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(204)
}
func certificateBytes(req *http.Request) []byte {
if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
return req.TLS.PeerCertificates[0].Raw
}
if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
bs := []byte(hdr)
// The certificate is in PEM format but with spaces for newlines. We
// need to reinstate the newlines for the PEM decoder. But we need to
// leave the spaces in the BEGIN and END lines - the first and last
// space - alone.
firstSpace := bytes.Index(bs, []byte(" "))
lastSpace := bytes.LastIndex(bs, []byte(" "))
for i := firstSpace + 1; i < lastSpace; i++ {
if bs[i] == ' ' {
bs[i] = '\n'
}
}
block, _ := pem.Decode(bs)
if block == nil {
// Decoding failed
return nil
}
return block.Bytes
}
return nil
}

View File

@@ -1,324 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"context"
"crypto/tls"
"encoding/binary"
"fmt"
io "io"
"log"
"net"
"time"
"github.com/syncthing/syncthing/lib/protocol"
)
const (
replicationReadTimeout = time.Minute
replicationWriteTimeout = 30 * time.Second
replicationHeartbeatInterval = time.Second * 30
)
type replicator interface {
send(key string, addrs []DatabaseAddress, seen int64)
}
// a replicationSender tries to connect to the remote address and provide
// them with a feed of replication updates.
type replicationSender struct {
dst string
cert tls.Certificate // our certificate
allowedIDs []protocol.DeviceID
outbox chan ReplicationRecord
}
func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
return &replicationSender{
dst: dst,
cert: cert,
allowedIDs: allowedIDs,
outbox: make(chan ReplicationRecord, replicationOutboxSize),
}
}
func (s *replicationSender) Serve(ctx context.Context) error {
// Sleep a little at startup. Peers often restart at the same time, and
// this avoid the service failing and entering backoff state
// unnecessarily, while also reducing the reconnect rate to something
// reasonable by default.
time.Sleep(2 * time.Second)
tlsCfg := &tls.Config{
Certificates: []tls.Certificate{s.cert},
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true,
}
// Dial the TLS connection.
conn, err := tls.Dial("tcp", s.dst, tlsCfg)
if err != nil {
log.Println("Replication connect:", err)
return err
}
defer func() {
conn.SetWriteDeadline(time.Now().Add(time.Second))
conn.Close()
}()
// The replication stream is not especially latency sensitive, but it is
// quite a lot of data in small writes. Make it more efficient.
if tcpc, ok := conn.NetConn().(*net.TCPConn); ok {
_ = tcpc.SetNoDelay(false)
}
// Get the other side device ID.
remoteID, err := deviceID(conn)
if err != nil {
log.Println("Replication connect:", err)
return err
}
// Verify it's in the set of allowed device IDs.
if !deviceIDIn(remoteID, s.allowedIDs) {
log.Println("Replication connect: unexpected device ID:", remoteID)
return err
}
heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
defer heartBeatTicker.Stop()
// Send records.
buf := make([]byte, 1024)
for {
select {
case <-heartBeatTicker.C:
if len(s.outbox) > 0 {
// No need to send heartbeats if there are events/prevrious
// heartbeats to send, they will keep the connection alive.
continue
}
// Empty replication message is the heartbeat:
s.outbox <- ReplicationRecord{}
case rec := <-s.outbox:
// Buffer must hold record plus four bytes for size
size := rec.Size()
if len(buf) < size+4 {
buf = make([]byte, size+4)
}
// Record comes after the four bytes size
n, err := rec.MarshalTo(buf[4:])
if err != nil {
// odd to get an error here, but we haven't sent anything
// yet so it's not fatal
replicationSendsTotal.WithLabelValues("error").Inc()
log.Println("Replication marshal:", err)
continue
}
binary.BigEndian.PutUint32(buf, uint32(n))
// Send
conn.SetWriteDeadline(time.Now().Add(replicationWriteTimeout))
if _, err := conn.Write(buf[:4+n]); err != nil {
replicationSendsTotal.WithLabelValues("error").Inc()
log.Println("Replication write:", err)
// Yes, we are losing the replication event here.
return err
}
replicationSendsTotal.WithLabelValues("success").Inc()
case <-ctx.Done():
return nil
}
}
}
func (s *replicationSender) String() string {
return fmt.Sprintf("replicationSender(%q)", s.dst)
}
func (s *replicationSender) send(key string, ps []DatabaseAddress, _ int64) {
item := ReplicationRecord{
Key: key,
Addresses: ps,
}
// The send should never block. The inbox is suitably buffered for at
// least a few seconds of stalls, which shouldn't happen in practice.
select {
case s.outbox <- item:
default:
replicationSendsTotal.WithLabelValues("drop").Inc()
}
}
// a replicationMultiplexer sends to multiple replicators
type replicationMultiplexer []replicator
func (m replicationMultiplexer) send(key string, ps []DatabaseAddress, seen int64) {
for _, s := range m {
// each send is nonblocking
s.send(key, ps, seen)
}
}
// replicationListener accepts incoming connections and reads replication
// items from them. Incoming items are applied to the KV store.
type replicationListener struct {
addr string
cert tls.Certificate
allowedIDs []protocol.DeviceID
db database
}
func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
return &replicationListener{
addr: addr,
cert: cert,
allowedIDs: allowedIDs,
db: db,
}
}
func (l *replicationListener) Serve(ctx context.Context) error {
tlsCfg := &tls.Config{
Certificates: []tls.Certificate{l.cert},
ClientAuth: tls.RequestClientCert,
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true,
}
lst, err := tls.Listen("tcp", l.addr, tlsCfg)
if err != nil {
log.Println("Replication listen:", err)
return err
}
defer lst.Close()
for {
select {
case <-ctx.Done():
return nil
default:
}
// Accept a connection
conn, err := lst.Accept()
if err != nil {
log.Println("Replication accept:", err)
return err
}
// Figure out the other side device ID
remoteID, err := deviceID(conn.(*tls.Conn))
if err != nil {
log.Println("Replication accept:", err)
conn.SetWriteDeadline(time.Now().Add(time.Second))
conn.Close()
continue
}
// Verify it is in the set of allowed device IDs
if !deviceIDIn(remoteID, l.allowedIDs) {
log.Println("Replication accept: unexpected device ID:", remoteID)
conn.SetWriteDeadline(time.Now().Add(time.Second))
conn.Close()
continue
}
go l.handle(ctx, conn)
}
}
func (l *replicationListener) String() string {
return fmt.Sprintf("replicationListener(%q)", l.addr)
}
func (l *replicationListener) handle(ctx context.Context, conn net.Conn) {
defer func() {
conn.SetWriteDeadline(time.Now().Add(time.Second))
conn.Close()
}()
buf := make([]byte, 1024)
for {
select {
case <-ctx.Done():
return
default:
}
conn.SetReadDeadline(time.Now().Add(replicationReadTimeout))
// First four bytes are the size
if _, err := io.ReadFull(conn, buf[:4]); err != nil {
log.Println("Replication read size:", err)
replicationRecvsTotal.WithLabelValues("error").Inc()
return
}
// Read the rest of the record
size := int(binary.BigEndian.Uint32(buf[:4]))
if len(buf) < size {
buf = make([]byte, size)
}
if size == 0 {
// Heartbeat, ignore
continue
}
if _, err := io.ReadFull(conn, buf[:size]); err != nil {
log.Println("Replication read record:", err)
replicationRecvsTotal.WithLabelValues("error").Inc()
return
}
// Unmarshal
var rec ReplicationRecord
if err := rec.Unmarshal(buf[:size]); err != nil {
log.Println("Replication unmarshal:", err)
replicationRecvsTotal.WithLabelValues("error").Inc()
continue
}
// Store
l.db.merge(rec.Key, rec.Addresses, rec.Seen)
replicationRecvsTotal.WithLabelValues("success").Inc()
}
}
func deviceID(conn *tls.Conn) (protocol.DeviceID, error) {
// Handshake may not be complete on the server side yet, which we need
// to get the client certificate.
if !conn.ConnectionState().HandshakeComplete {
if err := conn.Handshake(); err != nil {
return protocol.DeviceID{}, err
}
}
// We expect exactly one certificate.
certs := conn.ConnectionState().PeerCertificates
if len(certs) != 1 {
return protocol.DeviceID{}, fmt.Errorf("unexpected number of certificates (%d != 1)", len(certs))
}
return protocol.NewDeviceID(certs[0].Raw), nil
}
func deviceIDIn(id protocol.DeviceID, ids []protocol.DeviceID) bool {
for _, candidate := range ids {
if id == candidate {
return true
}
}
return false
}

View File

@@ -1,4 +0,0 @@
#!/bin/bash
addgroup --system syncthing
adduser --system --home /var/lib/syncthing-discosrv --ingroup syncthing syncthing-discosrv

View File

@@ -1,141 +1,141 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
package main
import (
"bytes"
"database/sql"
"fmt"
"io/ioutil"
"log"
"os"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"sync/atomic"
"time"
)
var (
buildInfo = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "build_info",
Help: "A metric with a constant '1' value labeled by version, goversion, builduser and builddate from which stdiscosrv was built.",
}, []string{"version", "goversion", "builduser", "builddate"})
type stats struct {
// Incremented atomically
announces int64
queries int64
answers int64
errors int64
}
apiRequestsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "api_requests_total",
Help: "Number of API requests.",
}, []string{"type", "result"})
apiRequestsSeconds = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "api_requests_seconds",
Help: "Latency of API requests.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
}, []string{"type"})
func (s *stats) Announce() {
atomic.AddInt64(&s.announces, 1)
}
lookupRequestsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "lookup_requests_total",
Help: "Number of lookup requests.",
}, []string{"result"})
announceRequestsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "announcement_requests_total",
Help: "Number of announcement requests.",
}, []string{"result"})
func (s *stats) Query() {
atomic.AddInt64(&s.queries, 1)
}
replicationSendsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "replication_sends_total",
Help: "Number of replication sends.",
}, []string{"result"})
replicationRecvsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "replication_recvs_total",
Help: "Number of replication receives.",
}, []string{"result"})
func (s *stats) Answer() {
atomic.AddInt64(&s.answers, 1)
}
databaseKeys = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "database_keys",
Help: "Number of database keys at last count.",
}, []string{"category"})
databaseStatisticsSeconds = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "database_statistics_seconds",
Help: "Time spent running the statistics routine.",
})
func (s *stats) Error() {
atomic.AddInt64(&s.errors, 1)
}
databaseOperations = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "database_operations_total",
Help: "Number of database operations.",
}, []string{"operation", "result"})
databaseOperationSeconds = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "database_operation_seconds",
Help: "Latency of database operations.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
}, []string{"operation"})
retryAfterHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "retry_after_seconds",
Help: "Retry-After header value in seconds.",
Buckets: prometheus.ExponentialBuckets(60, 2, 7), // 60, 120, 240, 480, 960, 1920, 3840
})
)
const (
dbOpGet = "get"
dbOpPut = "put"
dbOpMerge = "merge"
dbOpDelete = "delete"
dbResSuccess = "success"
dbResNotFound = "not_found"
dbResError = "error"
dbResUnmarshalError = "unmarsh_err"
)
func init() {
prometheus.MustRegister(buildInfo,
apiRequestsTotal, apiRequestsSeconds,
lookupRequestsTotal, announceRequestsTotal,
replicationSendsTotal, replicationRecvsTotal,
databaseKeys, databaseStatisticsSeconds,
databaseOperations, databaseOperationSeconds,
retryAfterHistogram)
processCollectorOpts := collectors.ProcessCollectorOpts{
Namespace: "syncthing_discovery",
PidFn: func() (int, error) {
return os.Getpid(), nil
},
// Reset returns a copy of the current stats and resets the counters to
// zero.
func (s *stats) Reset() stats {
// Create a copy of the stats using atomic reads
copy := stats{
announces: atomic.LoadInt64(&s.announces),
queries: atomic.LoadInt64(&s.queries),
answers: atomic.LoadInt64(&s.answers),
errors: atomic.LoadInt64(&s.errors),
}
prometheus.MustRegister(
collectors.NewProcessCollector(processCollectorOpts),
)
// Reset the stats by subtracting the values that we copied
atomic.AddInt64(&s.announces, -copy.announces)
atomic.AddInt64(&s.queries, -copy.queries)
atomic.AddInt64(&s.answers, -copy.answers)
atomic.AddInt64(&s.errors, -copy.errors)
return copy
}
type statssrv struct {
intv time.Duration
file string
db *sql.DB
}
func (s *statssrv) Serve() {
lastReset := time.Now()
for {
time.Sleep(next(s.intv))
stats := globalStats.Reset()
d := time.Since(lastReset).Seconds()
lastReset = time.Now()
log.Printf("Stats: %.02f announces/s, %.02f queries/s, %.02f answers/s, %.02f errors/s",
float64(stats.announces)/d, float64(stats.queries)/d, float64(stats.answers)/d, float64(stats.errors)/d)
if s.file != "" {
s.writeToFile(stats, d)
}
}
}
func (s *statssrv) Stop() {
panic("stop unimplemented")
}
func (s *statssrv) writeToFile(stats stats, secs float64) {
newLine := []byte("\n")
var addrs int
row := s.db.QueryRow("SELECT COUNT(*) FROM Addresses")
if err := row.Scan(&addrs); err != nil {
log.Println("stats query:", err)
return
}
fd, err := os.OpenFile(s.file, os.O_RDWR|os.O_CREATE, 0666)
if err != nil {
log.Println("stats file:", err)
return
}
defer func() {
err = fd.Close()
if err != nil {
log.Println("stats file:", err)
}
}()
bs, err := ioutil.ReadAll(fd)
if err != nil {
log.Println("stats file:", err)
return
}
lines := bytes.Split(bytes.TrimSpace(bs), newLine)
if len(lines) > 12 {
lines = lines[len(lines)-12:]
}
latest := fmt.Sprintf("%v: %6d addresses, %8.02f announces/s, %8.02f queries/s, %8.02f answers/s, %8.02f errors/s\n",
time.Now().UTC().Format(time.RFC3339), addrs,
float64(stats.announces)/secs, float64(stats.queries)/secs, float64(stats.answers)/secs, float64(stats.errors)/secs)
lines = append(lines, []byte(latest))
_, err = fd.Seek(0, 0)
if err != nil {
log.Println("stats file:", err)
return
}
err = fd.Truncate(0)
if err != nil {
log.Println("stats file:", err)
return
}
_, err = fd.Write(bytes.Join(lines, newLine))
if err != nil {
log.Println("stats file:", err)
return
}
}

View File

@@ -14,8 +14,6 @@ import (
"net/http"
"os"
"time"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
)
type event struct {
@@ -30,21 +28,16 @@ func main() {
log.SetFlags(0)
target := flag.String("target", "localhost:8384", "Target Syncthing instance")
types := flag.String("types", "", "Filter for specific event types (comma-separated)")
apikey := flag.String("apikey", "", "Syncthing API key")
flag.Parse()
if *apikey == "" {
log.Fatal("Must give -apikey argument")
}
var eventsArg string
if len(*types) > 0 {
eventsArg = "&events=" + *types
}
since := 0
for {
req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d%s", *target, since, eventsArg), nil)
req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d", *target, since), nil)
if err != nil {
log.Fatal(err)
}

View File

@@ -13,7 +13,6 @@ import (
"os"
"path/filepath"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
)
@@ -69,8 +68,8 @@ func main() {
}
blockSize := int(fi.Size())
if *standardBlocks || blockSize < protocol.MinBlockSize {
blockSize = protocol.BlockSize(fi.Size())
if *standardBlocks || blockSize < protocol.BlockSize {
blockSize = protocol.BlockSize
}
bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
if err != nil {

View File

@@ -7,7 +7,6 @@
package main
import (
"context"
"crypto/tls"
"errors"
"flag"
@@ -16,10 +15,8 @@ import (
"os"
"time"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/discover"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
)
@@ -85,7 +82,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
}
func checkServer(deviceID protocol.DeviceID, server string) checkResult {
disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger, nil)
disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
if err != nil {
return checkResult{error: err}
}
@@ -97,7 +94,7 @@ func checkServer(deviceID protocol.DeviceID, server string) checkResult {
})
go func() {
addresses, err := disco.Lookup(context.Background(), deviceID)
addresses, err := disco.Lookup(deviceID)
res <- checkResult{addresses: addresses, error: err}
}()

View File

@@ -1,45 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Command stfindignored lists ignored files under a given folder root.
package main
import (
"flag"
"fmt"
"os"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
)
func main() {
flag.Parse()
root := flag.Arg(0)
if root == "" {
root = "."
}
vfs := fs.NewWalkFilesystem(fs.NewFilesystem(fs.FilesystemTypeBasic, root))
ign := ignore.New(vfs)
if err := ign.Load(".stignore"); err != nil {
fmt.Fprintf(os.Stderr, "Fatal: loading ignores: %v\n", err)
os.Exit(1)
}
vfs.Walk(".", func(path string, info fs.FileInfo, err error) error {
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: %s: %v\n", path, err)
return fs.SkipDir
}
if ign.Match(path).IsIgnored() {
fmt.Println(path)
}
return nil
})
}

View File

@@ -15,8 +15,6 @@ import (
"os"
"path/filepath"
"time"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
)
func main() {
@@ -45,7 +43,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
}
p0 := filepath.Join(dir, string(n[0]), n[0:2])
err = os.MkdirAll(p0, 0o755)
err = os.MkdirAll(p0, 0755)
if err != nil {
log.Fatal(err)
}
@@ -68,7 +66,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
}
func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
src := io.LimitReader(&infiniteReader{fd}, s)
src := io.LimitReader(&inifiteReader{fd}, s)
dst, err := os.Create(p1)
if err != nil {
return err
@@ -84,7 +82,7 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
return err
}
os.Chmod(p1, os.FileMode(rand.Intn(0o777)|0o400))
_ = os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
return os.Chtimes(p1, t, t)
@@ -107,11 +105,11 @@ func readRand(bs []byte) (int, error) {
return len(bs), nil
}
type infiniteReader struct {
type inifiteReader struct {
rd io.ReadSeeker
}
func (i *infiniteReader) Read(bs []byte) (int, error) {
func (i *inifiteReader) Read(bs []byte) (int, error) {
n, err := i.rd.Read(bs)
if err == io.EOF {
err = nil

83
cmd/stindex/dump.go Normal file
View File

@@ -0,0 +1,83 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"encoding/binary"
"fmt"
"log"
"time"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/protocol"
)
func dump(ldb *db.Instance) {
it := ldb.NewIterator(nil, nil)
for it.Next() {
key := it.Key()
switch key[0] {
case db.KeyTypeDevice:
folder := binary.BigEndian.Uint32(key[1:])
device := binary.BigEndian.Uint32(key[1+4:])
name := nulString(key[1+4+4:])
fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
var f protocol.FileInfo
err := f.Unmarshal(it.Value())
if err != nil {
log.Fatal(err)
}
fmt.Printf(" V:%v\n", f)
case db.KeyTypeGlobal:
folder := binary.BigEndian.Uint32(key[1:])
name := nulString(key[1+4:])
var flv db.VersionList
flv.Unmarshal(it.Value())
fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
case db.KeyTypeBlock:
folder := binary.BigEndian.Uint32(key[1:])
hash := key[1+4 : 1+4+32]
name := nulString(key[1+4+32:])
fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
case db.KeyTypeDeviceStatistic:
fmt.Printf("[dstat] K:%x V:%x\n", it.Key(), it.Value())
case db.KeyTypeFolderStatistic:
fmt.Printf("[fstat] K:%x V:%x\n", it.Key(), it.Value())
case db.KeyTypeVirtualMtime:
folder := binary.BigEndian.Uint32(key[1:])
name := nulString(key[1+4:])
val := it.Value()
var real, virt time.Time
real.UnmarshalBinary(val[:len(val)/2])
virt.UnmarshalBinary(val[len(val)/2:])
fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, real, virt)
case db.KeyTypeFolderIdx:
key := binary.BigEndian.Uint32(it.Key()[1:])
fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
case db.KeyTypeDeviceIdx:
key := binary.BigEndian.Uint32(it.Key()[1:])
val := it.Value()
if len(val) == 0 {
fmt.Printf("[deviceidx] K:%d V:<nil>\n", key)
} else {
dev := protocol.DeviceIDFromBytes(val)
fmt.Printf("[deviceidx] K:%d V:%s\n", key, dev)
}
default:
fmt.Printf("[???]\n %x\n %x\n", it.Key(), it.Value())
}
}
}

94
cmd/stindex/dumpsize.go Normal file
View File

@@ -0,0 +1,94 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"container/heap"
"encoding/binary"
"fmt"
"github.com/syncthing/syncthing/lib/db"
)
type SizedElement struct {
key string
size int
}
type ElementHeap []SizedElement
func (h ElementHeap) Len() int { return len(h) }
func (h ElementHeap) Less(i, j int) bool { return h[i].size > h[j].size }
func (h ElementHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *ElementHeap) Push(x interface{}) {
*h = append(*h, x.(SizedElement))
}
func (h *ElementHeap) Pop() interface{} {
old := *h
n := len(old)
x := old[n-1]
*h = old[0 : n-1]
return x
}
func dumpsize(ldb *db.Instance) {
h := &ElementHeap{}
heap.Init(h)
it := ldb.NewIterator(nil, nil)
var ele SizedElement
for it.Next() {
key := it.Key()
switch key[0] {
case db.KeyTypeDevice:
folder := binary.BigEndian.Uint32(key[1:])
device := binary.BigEndian.Uint32(key[1+4:])
name := nulString(key[1+4+4:])
ele.key = fmt.Sprintf("DEVICE:%d:%d:%s", folder, device, name)
case db.KeyTypeGlobal:
folder := binary.BigEndian.Uint32(key[1:])
name := nulString(key[1+4:])
ele.key = fmt.Sprintf("GLOBAL:%d:%s", folder, name)
case db.KeyTypeBlock:
folder := binary.BigEndian.Uint32(key[1:])
hash := key[1+4 : 1+4+32]
name := nulString(key[1+4+32:])
ele.key = fmt.Sprintf("BLOCK:%d:%x:%s", folder, hash, name)
case db.KeyTypeDeviceStatistic:
ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])
case db.KeyTypeFolderStatistic:
ele.key = fmt.Sprintf("FOLDERSTATS:%s", key[1:])
case db.KeyTypeVirtualMtime:
ele.key = fmt.Sprintf("MTIME:%s", key[1:])
case db.KeyTypeFolderIdx:
id := binary.BigEndian.Uint32(key[1:])
ele.key = fmt.Sprintf("FOLDERIDX:%d", id)
case db.KeyTypeDeviceIdx:
id := binary.BigEndian.Uint32(key[1:])
ele.key = fmt.Sprintf("DEVICEIDX:%d", id)
default:
ele.key = fmt.Sprintf("UNKNOWN:%x", key)
}
ele.size = len(it.Value())
heap.Push(h, ele)
}
for h.Len() > 0 {
ele = heap.Pop(h).(SizedElement)
fmt.Println(ele.key, ele.size)
}
}

Some files were not shown because too many files have changed in this diff Show More