mirror of
https://github.com/syncthing/syncthing.git
synced 2026-01-31 09:11:26 -05:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4a2d7d4604 |
@@ -1,7 +1,7 @@
|
||||
version = 1
|
||||
|
||||
exclude_patterns = ["**/*.pb.go"]
|
||||
test_patterns = ["**/*_test.go"]
|
||||
exclude_patterns = ["*.pb.go"]
|
||||
test_patterns = ["*_test.go"]
|
||||
|
||||
[[analyzers]]
|
||||
name = "go"
|
||||
|
||||
16
.github/dependabot.yml
vendored
16
.github/dependabot.yml
vendored
@@ -1,13 +1,7 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 10
|
||||
|
||||
- package-ecosystem: "gomod"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 10
|
||||
- package-ecosystem: gomod
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 10
|
||||
|
||||
61
.github/workflows/build-infra-dockers.yaml
vendored
61
.github/workflows/build-infra-dockers.yaml
vendored
@@ -1,61 +0,0 @@
|
||||
name: Build Infrastructure Images
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- infrastructure
|
||||
|
||||
env:
|
||||
GO_VERSION: "~1.21.5"
|
||||
CGO_ENABLED: "0"
|
||||
BUILD_USER: docker
|
||||
BUILD_HOST: github.syncthing.net
|
||||
|
||||
jobs:
|
||||
docker-syncthing:
|
||||
name: Build and push Docker images
|
||||
runs-on: ubuntu-latest
|
||||
environment: docker
|
||||
strategy:
|
||||
matrix:
|
||||
pkg:
|
||||
- stcrashreceiver
|
||||
- strelaypoolsrv
|
||||
- stupgrades
|
||||
- ursrv
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
check-latest: true
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Build binaries
|
||||
run: |
|
||||
for arch in arm64 amd64; do
|
||||
go run build.go -goos linux -goarch "$arch" build ${{ matrix.pkg }}
|
||||
mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-"$arch"
|
||||
done
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.${{ matrix.pkg }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: syncthing/${{ matrix.pkg }}:latest,syncthing/${{ matrix.pkg }}:${{ github.sha }}
|
||||
labels: |
|
||||
org.opencontainers.image.revision=${{ github.sha }}
|
||||
830
.github/workflows/build-syncthing.yaml
vendored
830
.github/workflows/build-syncthing.yaml
vendored
@@ -1,830 +0,0 @@
|
||||
name: Build Syncthing
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
schedule:
|
||||
# Run nightly build at 05:00 UTC
|
||||
- cron: '00 05 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
# The go version to use for builds. We set check-latest to true when
|
||||
# installing, so we get the latest patch version that matches the
|
||||
# expression.
|
||||
GO_VERSION: "~1.21.5"
|
||||
|
||||
# Optimize compatibility on the slow archictures.
|
||||
GO386: softfloat
|
||||
GOARM: "5"
|
||||
GOMIPS: softfloat
|
||||
|
||||
# Avoid hilarious amounts of obscuring log output when running tests.
|
||||
LOGGER_DISCARD: "1"
|
||||
|
||||
# Our build metadata
|
||||
BUILD_USER: builder
|
||||
BUILD_HOST: github.syncthing.net
|
||||
|
||||
# A note on actions and third party code... The actions under actions/ (like
|
||||
# `uses: actions/checkout`) are maintained by GitHub, and we need to trust
|
||||
# GitHub to maintain their code and infrastructure or we're in deep shit in
|
||||
# general. The same doesn't necessarily apply to other actions authors, so
|
||||
# some care needs to be taken when adding steps, especially in the paths
|
||||
# that lead up to code being packaged and signed.
|
||||
|
||||
jobs:
|
||||
|
||||
#
|
||||
# Tests for all platforms. Runs a matrix build on Windows, Linux and Mac,
|
||||
# with the list of expected supported Go versions (current, previous).
|
||||
#
|
||||
|
||||
build-test:
|
||||
name: Build and test
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: ["windows-latest", "ubuntu-latest", "macos-latest"]
|
||||
# The oldest version in this list should match what we have in our go.mod.
|
||||
# Variables don't seem to be supported here, or we could have done something nice.
|
||||
go: ["~1.20.12", "~1.21.5"]
|
||||
runs-on: ${{ matrix.runner }}
|
||||
steps:
|
||||
- name: Set git to use LF
|
||||
if: matrix.runner == 'windows-latest'
|
||||
# Without this, the Windows checkout will happen with CRLF line
|
||||
# endings, which is fine for the source code but messes up tests
|
||||
# that depend on data on disk being as expected. Ideally, those
|
||||
# tests should be fixed, but not today.
|
||||
run: |
|
||||
git config --global core.autocrlf false
|
||||
git config --global core.eol lf
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go }}
|
||||
cache: true
|
||||
check-latest: true
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
go run build.go
|
||||
|
||||
- name: Install go-test-json-to-loki
|
||||
run: |
|
||||
go install calmh.dev/go-test-json-to-loki@latest
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
go version
|
||||
go run build.go test | go-test-json-to-loki
|
||||
env:
|
||||
GOFLAGS: "-json"
|
||||
LOKI_URL: ${{ vars.LOKI_URL }}
|
||||
LOKI_USER: ${{ vars.LOKI_USER }}
|
||||
LOKI_PASSWORD: ${{ secrets.LOKI_PASSWORD }}
|
||||
LOKI_LABELS: "go=${{ matrix.go }},runner=${{ matrix.runner }},repo=${{ github.repository }},ref=${{ github.ref }}"
|
||||
|
||||
#
|
||||
# Meta checks for formatting, copyright, etc
|
||||
#
|
||||
|
||||
correctness:
|
||||
name: Check correctness
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- name: Check correctness
|
||||
run: |
|
||||
go test -v ./meta
|
||||
|
||||
#
|
||||
# The basic checks job is a virtual one that depends on the matrix tests,
|
||||
# the correctness checks, and various builds that we always do. This makes
|
||||
# it easy to have the PR process have a single test as a gatekeeper for
|
||||
# merging, instead of having to add all the matrix tests and update them
|
||||
# each time the version changes. (The top level test is not available for
|
||||
# choosing there, only the matrix "children".)
|
||||
#
|
||||
|
||||
basics:
|
||||
name: Basic checks passed
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- build-test
|
||||
- correctness
|
||||
- package-linux
|
||||
- package-cross
|
||||
- package-source
|
||||
- package-debian
|
||||
- govulncheck
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
#
|
||||
# Windows
|
||||
#
|
||||
|
||||
package-windows:
|
||||
name: Package for Windows
|
||||
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
|
||||
environment: signing
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- name: Set git to use LF
|
||||
# Without this, the checkout will happen with CRLF line endings,
|
||||
# which is fine for the source code but messes up tests that depend
|
||||
# on data on disk being as expected. Ideally, those tests should be
|
||||
# fixed, but not today.
|
||||
run: |
|
||||
git config --global core.autocrlf false
|
||||
git config --global core.eol lf
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- name: Get actual Go version
|
||||
run: |
|
||||
go version
|
||||
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
|
||||
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: |
|
||||
~\AppData\Local\go-build
|
||||
~\go\pkg\mod
|
||||
key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@v1.4.0
|
||||
|
||||
- name: Create packages
|
||||
run: |
|
||||
go run build.go -goarch amd64 zip
|
||||
go run build.go -goarch arm zip
|
||||
go run build.go -goarch arm64 zip
|
||||
go run build.go -goarch 386 zip
|
||||
env:
|
||||
CGO_ENABLED: "0"
|
||||
CODESIGN_SIGNTOOL: ${{ secrets.CODESIGN_SIGNTOOL }}
|
||||
CODESIGN_CERTIFICATE_BASE64: ${{ secrets.CODESIGN_CERTIFICATE_BASE64 }}
|
||||
CODESIGN_CERTIFICATE_PASSWORD: ${{ secrets.CODESIGN_CERTIFICATE_PASSWORD }}
|
||||
CODESIGN_TIMESTAMP_SERVER: ${{ secrets.CODESIGN_TIMESTAMP_SERVER }}
|
||||
|
||||
- name: Archive artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: packages-windows
|
||||
path: syncthing-windows-*.zip
|
||||
|
||||
#
|
||||
# Linux
|
||||
#
|
||||
|
||||
package-linux:
|
||||
name: Package for Linux
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- name: Get actual Go version
|
||||
run: |
|
||||
go version
|
||||
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
|
||||
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
|
||||
|
||||
- name: Create packages
|
||||
run: |
|
||||
archs=$(go tool dist list | grep linux | sed 's#linux/##')
|
||||
for goarch in $archs ; do
|
||||
go run build.go -goarch "$goarch" tar
|
||||
done
|
||||
env:
|
||||
CGO_ENABLED: "0"
|
||||
|
||||
- name: Archive artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: packages-linux
|
||||
path: syncthing-linux-*.tar.gz
|
||||
|
||||
#
|
||||
# macOS
|
||||
#
|
||||
|
||||
package-macos:
|
||||
name: Package for macOS
|
||||
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
|
||||
environment: signing
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- name: Get actual Go version
|
||||
run: |
|
||||
go version
|
||||
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
|
||||
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-package-${{ hashFiles('**/go.sum') }}
|
||||
|
||||
- name: Import signing certificate
|
||||
run: |
|
||||
# Set up a run-specific keychain, making it available for the
|
||||
# `codesign` tool.
|
||||
umask 066
|
||||
KEYCHAIN_PATH=$RUNNER_TEMP/codesign.keychain
|
||||
KEYCHAIN_PASSWORD=$(uuidgen)
|
||||
security create-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
|
||||
security default-keychain -s "$KEYCHAIN_PATH"
|
||||
security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH"
|
||||
security set-keychain-settings -lut 21600 "$KEYCHAIN_PATH"
|
||||
|
||||
# Import the certificate
|
||||
CERTIFICATE_PATH=$RUNNER_TEMP/codesign.p12
|
||||
echo "$DEVELOPER_ID_CERTIFICATE_BASE64" | base64 -d -o "$CERTIFICATE_PATH"
|
||||
security import "$CERTIFICATE_PATH" -k "$KEYCHAIN_PATH" -P "$DEVELOPER_ID_CERTIFICATE_PASSWORD" -T /usr/bin/codesign -T /usr/bin/productsign
|
||||
security set-key-partition-list -S apple-tool:,apple: -s -k actions "$KEYCHAIN_PATH"
|
||||
|
||||
# Set the codesign identity for following steps
|
||||
echo "CODESIGN_IDENTITY=$CODESIGN_IDENTITY" >> $GITHUB_ENV
|
||||
env:
|
||||
DEVELOPER_ID_CERTIFICATE_BASE64: ${{ secrets.DEVELOPER_ID_CERTIFICATE_BASE64 }}
|
||||
DEVELOPER_ID_CERTIFICATE_PASSWORD: ${{ secrets.DEVELOPER_ID_CERTIFICATE_PASSWORD }}
|
||||
CODESIGN_IDENTITY: ${{ secrets.CODESIGN_IDENTITY }}
|
||||
|
||||
- name: Create package (amd64)
|
||||
run: |
|
||||
go run build.go -goarch amd64 zip
|
||||
env:
|
||||
CGO_ENABLED: "1"
|
||||
|
||||
- name: Create package (arm64 cross)
|
||||
run: |
|
||||
cat <<EOT > xgo.sh
|
||||
#!/bin/bash
|
||||
CGO_ENABLED=1 \
|
||||
CGO_CFLAGS="-target arm64-apple-macos10.15" \
|
||||
CGO_LDFLAGS="-target arm64-apple-macos10.15" \
|
||||
go "\$@"
|
||||
EOT
|
||||
chmod 755 xgo.sh
|
||||
go run build.go -gocmd ./xgo.sh -goarch arm64 zip
|
||||
env:
|
||||
CGO_ENABLED: "1"
|
||||
|
||||
- name: Create package (universal)
|
||||
run: |
|
||||
rm -rf _tmp
|
||||
mkdir _tmp
|
||||
pushd _tmp
|
||||
|
||||
unzip ../syncthing-macos-amd64-*.zip
|
||||
unzip ../syncthing-macos-arm64-*.zip
|
||||
lipo -create syncthing-macos-amd64-*/syncthing syncthing-macos-arm64-*/syncthing -o syncthing
|
||||
|
||||
amd64=(syncthing-macos-amd64-*)
|
||||
universal="${amd64/amd64/universal}"
|
||||
mv "$amd64" "$universal"
|
||||
mv syncthing "$universal"
|
||||
zip -r "../$universal.zip" "$universal"
|
||||
|
||||
- name: Archive artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: packages-macos
|
||||
path: syncthing-*.zip
|
||||
|
||||
notarize-macos:
|
||||
name: Notarize for macOS
|
||||
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
|
||||
environment: signing
|
||||
needs:
|
||||
- package-macos
|
||||
- basics
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: packages-macos
|
||||
|
||||
- name: Notarize binaries
|
||||
run: |
|
||||
APPSTORECONNECT_API_KEY_PATH="$RUNNER_TEMP/apikey.p8"
|
||||
echo "$APPSTORECONNECT_API_KEY" | base64 -d -o "$APPSTORECONNECT_API_KEY_PATH"
|
||||
for file in syncthing-macos-*.zip ; do
|
||||
xcrun notarytool submit \
|
||||
-k "$APPSTORECONNECT_API_KEY_PATH" \
|
||||
-d "$APPSTORECONNECT_API_KEY_ID" \
|
||||
-i "$APPSTORECONNECT_API_KEY_ISSUER" \
|
||||
$file
|
||||
done
|
||||
env:
|
||||
APPSTORECONNECT_API_KEY: ${{ secrets.APPSTORECONNECT_API_KEY }}
|
||||
APPSTORECONNECT_API_KEY_ID: ${{ secrets.APPSTORECONNECT_API_KEY_ID }}
|
||||
APPSTORECONNECT_API_KEY_ISSUER: ${{ secrets.APPSTORECONNECT_API_KEY_ISSUER }}
|
||||
|
||||
#
|
||||
# Cross compile other unixes
|
||||
#
|
||||
|
||||
package-cross:
|
||||
name: Package cross compiled
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- name: Get actual Go version
|
||||
run: |
|
||||
go version
|
||||
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
|
||||
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-cross-${{ hashFiles('**/go.sum') }}
|
||||
|
||||
- name: Create packages
|
||||
run: |
|
||||
platforms=$(go tool dist list \
|
||||
| grep -v aix/ppc64 \
|
||||
| grep -v android/ \
|
||||
| grep -v darwin/ \
|
||||
| grep -v ios/ \
|
||||
| grep -v js/ \
|
||||
| grep -v linux/ \
|
||||
| grep -v nacl/ \
|
||||
| grep -v plan9/ \
|
||||
| grep -v windows/ \
|
||||
| grep -v /wasm \
|
||||
)
|
||||
|
||||
# Build for each platform with errors silenced, because we expect
|
||||
# some oddball platforms to fail. This avoids a bunch of errors in
|
||||
# the GitHub Actions output, instead summarizing each build
|
||||
# failure as a warning.
|
||||
for plat in $platforms; do
|
||||
goos="${plat%/*}"
|
||||
goarch="${plat#*/}"
|
||||
echo "::group ::$plat"
|
||||
if ! go run build.go -goos "$goos" -goarch "$goarch" tar 2>/dev/null; then
|
||||
echo "::warning ::Failed to build for $plat"
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
done
|
||||
env:
|
||||
CGO_ENABLED: "0"
|
||||
|
||||
- name: Archive artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: packages-other
|
||||
path: syncthing-*.tar.gz
|
||||
|
||||
#
|
||||
# Source
|
||||
#
|
||||
|
||||
package-source:
|
||||
name: Package source code
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- name: Package source
|
||||
run: |
|
||||
version=$(go run build.go version)
|
||||
echo "$version" > RELEASE
|
||||
|
||||
go mod vendor
|
||||
go run build.go assets
|
||||
|
||||
cd ..
|
||||
|
||||
tar c -z -f "syncthing-source-$version.tar.gz" \
|
||||
--exclude .git \
|
||||
syncthing
|
||||
|
||||
mv "syncthing-source-$version.tar.gz" syncthing
|
||||
|
||||
- name: Archive artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: packages-source
|
||||
path: syncthing-source-*.tar.gz
|
||||
|
||||
#
|
||||
# Sign binaries for auto upgrade, generate ASC signature files
|
||||
#
|
||||
|
||||
sign-for-upgrade:
|
||||
name: Sign for upgrade
|
||||
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/heads/release-'))
|
||||
environment: signing
|
||||
needs:
|
||||
- basics
|
||||
- package-windows
|
||||
- package-linux
|
||||
- package-macos
|
||||
- package-cross
|
||||
- package-source
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
repository: syncthing/release-tools
|
||||
path: tools
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
|
||||
- name: Install signing tool
|
||||
run: |
|
||||
go install ./cmd/stsigtool
|
||||
|
||||
- name: Sign archives
|
||||
run: |
|
||||
export PRIVATE_KEY="$RUNNER_TEMP/privkey.pem"
|
||||
export PATH="$PATH:$(go env GOPATH)/bin"
|
||||
echo "$STSIGTOOL_PRIVATE_KEY" | base64 -d > "$PRIVATE_KEY"
|
||||
mkdir packages
|
||||
mv packages-*/* packages
|
||||
pushd packages
|
||||
"$GITHUB_WORKSPACE/tools/sign-only"
|
||||
rm -f "$PRIVATE_KEY"
|
||||
env:
|
||||
STSIGTOOL_PRIVATE_KEY: ${{ secrets.STSIGTOOL_PRIVATE_KEY }}
|
||||
|
||||
- name: Create and sign .asc files
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt -y install gnupg
|
||||
|
||||
export SIGNING_KEY="$RUNNER_TEMP/gpg-secret.asc"
|
||||
echo "$GNUPG_SIGNING_KEY_BASE64" | base64 -d > "$SIGNING_KEY"
|
||||
gpg --import < "$SIGNING_KEY"
|
||||
|
||||
pushd packages
|
||||
files=(*.tar.gz *.zip)
|
||||
sha1sum "${files[@]}" | gpg --clearsign > sha1sum.txt.asc
|
||||
sha256sum "${files[@]}" | gpg --clearsign > sha256sum.txt.asc
|
||||
gpg --sign --armour --detach syncthing-source-*.tar.gz
|
||||
popd
|
||||
rm -f "$SIGNING_KEY" .gnupg
|
||||
env:
|
||||
GNUPG_SIGNING_KEY_BASE64: ${{ secrets.GNUPG_SIGNING_KEY_BASE64 }}
|
||||
|
||||
- name: Archive artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: packages-signed
|
||||
path: packages/*
|
||||
|
||||
#
|
||||
# Debian
|
||||
#
|
||||
|
||||
package-debian:
|
||||
name: Package for Debian
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- name: Get actual Go version
|
||||
run: |
|
||||
go version
|
||||
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
|
||||
|
||||
- uses: ruby/setup-ruby@v1
|
||||
with:
|
||||
ruby-version: '3.0'
|
||||
|
||||
- name: Install fpm
|
||||
run: |
|
||||
gem install fpm
|
||||
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-debian-${{ hashFiles('**/go.sum') }}
|
||||
|
||||
- name: Package for Debian
|
||||
run: |
|
||||
for arch in amd64 i386 armhf armel arm64 ; do
|
||||
go run build.go -no-upgrade -installsuffix=no-upgrade -goarch "$arch" deb
|
||||
done
|
||||
env:
|
||||
BUILD_USER: debian
|
||||
|
||||
- name: Archive artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: debian-packages
|
||||
path: "*.deb"
|
||||
|
||||
#
|
||||
# Nightlies
|
||||
#
|
||||
|
||||
publish-nightly:
|
||||
name: Publish nightly build
|
||||
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && startsWith(github.ref, 'refs/heads/release-nightly')
|
||||
environment: signing
|
||||
needs:
|
||||
- sign-for-upgrade
|
||||
- notarize-macos
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
repository: syncthing/release-tools
|
||||
path: tools
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: packages-signed
|
||||
path: packages
|
||||
|
||||
- name: Create release json
|
||||
run: |
|
||||
cd packages
|
||||
"$GITHUB_WORKSPACE/tools/generate-release-json" "$BASE_URL" > nightly.json
|
||||
env:
|
||||
BASE_URL: https://syncthing.ams3.digitaloceanspaces.com/nightly/
|
||||
|
||||
- name: Push artifacts
|
||||
uses: docker://docker.io/rclone/rclone:latest
|
||||
env:
|
||||
RCLONE_CONFIG_SPACES_TYPE: s3
|
||||
RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
|
||||
RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
|
||||
RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
|
||||
RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
|
||||
RCLONE_CONFIG_SPACES_ACL: public-read
|
||||
with:
|
||||
args: sync packages spaces:syncthing/nightly
|
||||
|
||||
#
|
||||
# Push release artifacts to Spaces
|
||||
#
|
||||
|
||||
publish-release-files:
|
||||
name: Publish release files
|
||||
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/release'
|
||||
environment: signing
|
||||
needs:
|
||||
- sign-for-upgrade
|
||||
- package-debian
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download signed packages
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: packages-signed
|
||||
path: packages
|
||||
|
||||
- name: Download debian packages
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: debian-packages
|
||||
path: packages
|
||||
|
||||
- name: Set version
|
||||
run: |
|
||||
version=$(go run build.go version)
|
||||
echo "VERSION=$version" >> $GITHUB_ENV
|
||||
|
||||
- name: Push to Spaces (${{ env.VERSION }})
|
||||
uses: docker://docker.io/rclone/rclone:latest
|
||||
env:
|
||||
RCLONE_CONFIG_SPACES_TYPE: s3
|
||||
RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
|
||||
RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
|
||||
RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
|
||||
RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
|
||||
RCLONE_CONFIG_SPACES_ACL: public-read
|
||||
with:
|
||||
args: sync packages spaces:syncthing/release/${{ env.VERSION }}
|
||||
|
||||
- name: Push to Spaces (latest)
|
||||
uses: docker://docker.io/rclone/rclone:latest
|
||||
env:
|
||||
RCLONE_CONFIG_SPACES_TYPE: s3
|
||||
RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean
|
||||
RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }}
|
||||
RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }}
|
||||
RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com
|
||||
RCLONE_CONFIG_SPACES_ACL: public-read
|
||||
with:
|
||||
args: sync spaces:syncthing/release/${{ env.VERSION }} spaces:syncthing/release/latest
|
||||
|
||||
#
|
||||
# Build and push to Docker Hub
|
||||
#
|
||||
|
||||
docker-syncthing:
|
||||
name: Build and push Docker images
|
||||
runs-on: ubuntu-latest
|
||||
if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/release' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/infrastructure' || startsWith(github.ref, 'refs/heads/release-'))
|
||||
environment: docker
|
||||
strategy:
|
||||
matrix:
|
||||
pkg:
|
||||
- syncthing
|
||||
- strelaysrv
|
||||
- stdiscosrv
|
||||
include:
|
||||
- pkg: syncthing
|
||||
dockerfile: Dockerfile
|
||||
image: syncthing/syncthing
|
||||
- pkg: strelaysrv
|
||||
dockerfile: Dockerfile.strelaysrv
|
||||
image: syncthing/relaysrv
|
||||
- pkg: stdiscosrv
|
||||
dockerfile: Dockerfile.stdiscosrv
|
||||
image: syncthing/discosrv
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- name: Get actual Go version
|
||||
run: |
|
||||
go version
|
||||
echo "GO_VERSION=$(go version | sed 's#^.*go##;s# .*##')" >> $GITHUB_ENV
|
||||
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: |
|
||||
~/.cache/go-build
|
||||
~/go/pkg/mod
|
||||
key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-docker-${{ matrix.pkg }}-${{ hashFiles('**/go.sum') }}
|
||||
|
||||
- name: Build binaries
|
||||
run: |
|
||||
for arch in amd64 arm64 arm; do
|
||||
go run build.go -goos linux -goarch "$arch" -no-upgrade build ${{ matrix.pkg }}
|
||||
mv ${{ matrix.pkg }} ${{ matrix.pkg }}-linux-"$arch"
|
||||
done
|
||||
env:
|
||||
CGO_ENABLED: "0"
|
||||
BUILD_USER: docker
|
||||
|
||||
- name: Check if we will be able to push images
|
||||
run: |
|
||||
if [[ "${{ secrets.DOCKERHUB_TOKEN }}" != "" ]]; then
|
||||
echo "DOCKER_PUSH=true" >> $GITHUB_ENV;
|
||||
fi
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
if: env.DOCKER_PUSH == 'true'
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Set version tags
|
||||
run: |
|
||||
version=$(go run build.go version)
|
||||
version=${version#v}
|
||||
if [[ $version == @([0-9]|[0-9][0-9]).@([0-9]|[0-9][0-9]).@([0-9]|[0-9][0-9]) ]] ; then
|
||||
echo Release version, pushing to :latest and version tags
|
||||
major=${version%.*.*}
|
||||
minor=${version%.*}
|
||||
tags=${{ matrix.image }}:$version,${{ matrix.image }}:$major,${{ matrix.image }}:$minor,${{ matrix.image }}:latest
|
||||
elif [[ $version == *-rc.@([0-9]|[0-9][0-9]) ]] ; then
|
||||
echo Release candidate, pushing to :rc
|
||||
tags=${{ matrix.image }}:rc
|
||||
else
|
||||
echo Development version, pushing to :edge
|
||||
tags=${{ matrix.image }}:edge
|
||||
fi
|
||||
echo "DOCKER_TAGS=$tags" >> $GITHUB_ENV
|
||||
echo "VERSION=$version" >> $GITHUB_ENV
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ${{ matrix.dockerfile }}
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/7
|
||||
push: ${{ env.DOCKER_PUSH == 'true' }}
|
||||
tags: ${{ env.DOCKER_TAGS }}
|
||||
labels: |
|
||||
org.opencontainers.image.version=${{ env.VERSION }}
|
||||
org.opencontainers.image.revision=${{ github.sha }}
|
||||
|
||||
#
|
||||
# Check for known vulnerabilities in Go dependencies
|
||||
#
|
||||
|
||||
govulncheck:
|
||||
runs-on: ubuntu-latest
|
||||
name: Run govulncheck
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache: false
|
||||
check-latest: true
|
||||
|
||||
- name: run govulncheck
|
||||
run: |
|
||||
go run build.go assets
|
||||
go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
govulncheck ./...
|
||||
21
.github/workflows/trigger-nightly.yaml
vendored
21
.github/workflows/trigger-nightly.yaml
vendored
@@ -1,21 +0,0 @@
|
||||
name: Trigger nightly build & release
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# Run nightly build at 01:00 UTC
|
||||
- cron: '00 01 * * *'
|
||||
|
||||
jobs:
|
||||
|
||||
trigger-nightly:
|
||||
runs-on: ubuntu-latest
|
||||
name: Push to release-nightly to trigger build
|
||||
steps:
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
|
||||
fetch-depth: 0
|
||||
|
||||
- run: |
|
||||
git push origin main:release-nightly
|
||||
28
.github/workflows/update-docs-translations.yaml
vendored
28
.github/workflows/update-docs-translations.yaml
vendored
@@ -1,28 +0,0 @@
|
||||
name: Update translations and documentation
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '42 3 * * 1'
|
||||
|
||||
jobs:
|
||||
|
||||
update_transifex_docs:
|
||||
runs-on: ubuntu-latest
|
||||
name: Update translations and documentation
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ^1.19.6
|
||||
- run: |
|
||||
set -euo pipefail
|
||||
git config --global user.name 'Syncthing Release Automation'
|
||||
git config --global user.email 'release@syncthing.net'
|
||||
bash build.sh translate
|
||||
bash build.sh prerelease
|
||||
git push
|
||||
env:
|
||||
WEBLATE_TOKEN: ${{ secrets.WEBLATE_TOKEN }}
|
||||
4
.gitignore
vendored
4
.gitignore
vendored
@@ -1,10 +1,11 @@
|
||||
/syncthing
|
||||
/stdiscosrv
|
||||
syncthing.exe
|
||||
stdiscosrv.exe
|
||||
*.tar.gz
|
||||
*.zip
|
||||
*.asc
|
||||
*.deb
|
||||
*.exe
|
||||
.jshintrc
|
||||
coverage.out
|
||||
files/pidx
|
||||
@@ -18,3 +19,4 @@ deb
|
||||
/repos
|
||||
/proto/scripts/protoc-gen-gosyncthing
|
||||
/gui/next-gen-gui
|
||||
.idea
|
||||
|
||||
4
.yamlfmt
4
.yamlfmt
@@ -1,4 +0,0 @@
|
||||
line_ending: lf
|
||||
formatter:
|
||||
type: basic
|
||||
retain_line_breaks: true
|
||||
68
AUTHORS
68
AUTHORS
@@ -18,33 +18,25 @@ Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.nor
|
||||
Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
|
||||
Alan Pope <alan@popey.com>
|
||||
Alberto Donato <albertodonato@users.noreply.github.com>
|
||||
Aleksey Vasenev <margtu-fivt@ya.ru>
|
||||
Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
|
||||
Alex Lindeman <139387+aelindeman@users.noreply.github.com>
|
||||
Alex Xu <alex.hello71@gmail.com>
|
||||
Alexander Graf (alex2108) <register-github@alex-graf.de>
|
||||
Alexander Seiler <seileralex@gmail.com>
|
||||
Alexandre Alves <alexandrealvesdb.contact@gmail.com>
|
||||
Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
|
||||
Aman Gupta <aman@tmm1.net>
|
||||
Anatoli Babenia <anatoli@rainforce.org>
|
||||
Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
|
||||
Andreas Sommer <andreas.sommer87@googlemail.com>
|
||||
andresvia <andres.via@gmail.com>
|
||||
Andrew Dunham (andrew-d) <andrew@du.nham.ca>
|
||||
Andrew Meyer <andrewm.bpi@gmail.com>
|
||||
Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
|
||||
Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
|
||||
André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
|
||||
andyleap <andyleap@gmail.com>
|
||||
Anjan Momi <anjan@momi.ca>
|
||||
Anthony Goeckner <agoeckner@users.noreply.github.com>
|
||||
Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
|
||||
Antony Male (canton7) <antony.male@gmail.com>
|
||||
Anur <anurnomeru@163.com>
|
||||
Aranjedeath <Aranjedeath@users.noreply.github.com>
|
||||
Arkadiusz Tymiński <gevleeog@gmail.com>
|
||||
Aroun <login@b-vo.fr>
|
||||
Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
|
||||
Artur Zubilewicz <AkaZecik@users.noreply.github.com>
|
||||
Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
|
||||
@@ -57,7 +49,6 @@ Ben Shepherd (benshep) <bjashepherd@gmail.com>
|
||||
Ben Sidhom (bsidhom) <bsidhom@gmail.com>
|
||||
Benedikt Heine (bebehei) <bebe@bebehei.de>
|
||||
Benedikt Morbach <benedikt.morbach@googlemail.com>
|
||||
Benjamin Nater <17193640+bn4t@users.noreply.github.com>
|
||||
Benno Fünfstück <benno.fuenfstueck@gmail.com>
|
||||
Benny Ng (tpng) <benny.tpng@gmail.com>
|
||||
boomsquared <54829195+boomsquared@users.noreply.github.com>
|
||||
@@ -69,75 +60,55 @@ Brian R. Becker (brbecker) <brbecker@gmail.com>
|
||||
bt90 <btom1990@googlemail.com>
|
||||
Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
|
||||
Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
|
||||
Catfriend1 <16361913+Catfriend1@users.noreply.github.com>
|
||||
Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
|
||||
Cedric Staniewski (xduugu) <cedric@gmx.ca>
|
||||
chenrui <rui@meetup.com>
|
||||
Chih-Hsuan Yen <yan12125@gmail.com> <1937689+yan12125@users.noreply.github.com>
|
||||
Chih-Hsuan Yen <yan12125@gmail.com>
|
||||
Choongkyu <choongkyu.kim+gh@gmail.com> <vapidlyrapid+gh@gmail.com>
|
||||
Chris Howie (cdhowie) <me@chrishowie.com>
|
||||
Chris Joel (cdata) <chris@scriptolo.gy>
|
||||
Chris Tonkinson <chris@masterbran.ch>
|
||||
Christian Kujau <ckujau@users.noreply.github.com>
|
||||
Christian Prescott <me@christianprescott.com>
|
||||
chucic <chucic@seznam.cz>
|
||||
cjc7373 <niuchangcun@gmail.com>
|
||||
Colin Kennedy (moshen) <moshen.colin@gmail.com>
|
||||
Cromefire_ <tim.l@nghorst.net> <26320625+cromefire@users.noreply.github.com>
|
||||
cui fliter <imcusg@gmail.com>
|
||||
Cyprien Devillez <cypx@users.noreply.github.com>
|
||||
d-volution <49024624+d-volution@users.noreply.github.com>
|
||||
Dale Visser <dale.visser@live.com>
|
||||
Dan <benda.daniel@gmail.com>
|
||||
Daniel Barczyk <46358936+DanielBarczyk@users.noreply.github.com>
|
||||
Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
|
||||
Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
|
||||
Daniel Martí (mvdan) <mvdan@mvdan.cc>
|
||||
Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
|
||||
David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
|
||||
deepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com>
|
||||
DeflateAwning <11021263+DeflateAwning@users.noreply.github.com>
|
||||
Denis A. (dva) <denisva@gmail.com>
|
||||
Dennis Wilson (snnd) <dw@risu.io>
|
||||
dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com> <27856297+dependabot-preview[bot]@users.noreply.github.com>
|
||||
dependabot[bot] <dependabot[bot]@users.noreply.github.com> <49699333+dependabot[bot]@users.noreply.github.com>
|
||||
derekriemer <derek.riemer@colorado.edu>
|
||||
DerRockWolf <50499906+DerRockWolf@users.noreply.github.com>
|
||||
desbma <desbma@users.noreply.github.com>
|
||||
Devon G. Redekopp <devon@redekopp.com>
|
||||
digital <didev@dinid.net>
|
||||
Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com>
|
||||
Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
|
||||
Domenic Horner <domenic@tgxn.net>
|
||||
Dominik Heidler (asdil12) <dominik@heidler.eu>
|
||||
Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
|
||||
Elliot Huffman <thelich2@gmail.com>
|
||||
Emil Hessman (ceh) <emil@hessman.se>
|
||||
Emil Lundberg <emil@emlun.se>
|
||||
Eng Zer Jun <engzerjun@gmail.com>
|
||||
entity0xfe <109791748+entity0xfe@users.noreply.github.com> <entity0xfe@my.domain>
|
||||
Eric Lesiuta <elesiuta@gmail.com>
|
||||
Eric P <eric@kastelo.net>
|
||||
Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
|
||||
Evan Spensley <94762716+0evan@users.noreply.github.com>
|
||||
Evgeny Kuznetsov <evgeny@kuznetsov.md>
|
||||
Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
|
||||
Felix <53702818+f-eliks@users.noreply.github.com>
|
||||
Felix Ableitner (Nutomic) <me@nutomic.com>
|
||||
Felix Lampe <mail@flampe.de>
|
||||
Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
|
||||
Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
|
||||
Frank Isemann (fti7) <frank@isemann.name>
|
||||
Gahl Saraf <saraf.gahl@gmail.com> <gahl@raftt.io>
|
||||
Gahl Saraf <saraf.gahl@gmail.com>
|
||||
georgespatton <georgespatton@users.noreply.github.com>
|
||||
ghjklw <malo@jaffre.info>
|
||||
Gilli Sigurdsson (gillisig) <gilli@vx.is>
|
||||
Gleb Sinyavskiy <zhulik.gleb@gmail.com>
|
||||
Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
|
||||
greatroar <61184462+greatroar@users.noreply.github.com>
|
||||
Greg <gco@jazzhaiku.com>
|
||||
guangwu <guoguangwu@magic-shield.com>
|
||||
gudvinr <gudvinr@gmail.com>
|
||||
Han Boetes <han@boetes.org>
|
||||
HansK-p <42314815+HansK-p@users.noreply.github.com>
|
||||
Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
|
||||
@@ -145,7 +116,6 @@ Heiko Zuerker (Smiley73) <heiko@zuerker.org>
|
||||
Hugo Locurcio <hugo.locurcio@hugo.pro>
|
||||
Iain Barnett <iainspeed@gmail.com>
|
||||
Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
|
||||
ignacy123 <ignacy.buczek@onet.pl>
|
||||
Ikko Ashimine <eltociear@gmail.com>
|
||||
Ilya Brin <464157+ilyabrin@users.noreply.github.com>
|
||||
Iskander Sharipov (Alex) <quasilyte@gmail.com>
|
||||
@@ -155,16 +125,12 @@ Jack Croft <jccroft1@users.noreply.github.com>
|
||||
Jacob <jyundt@gmail.com>
|
||||
Jake Peterson (acogdev) <jake@acogdev.com>
|
||||
Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
|
||||
James O'Beirne <wild-github@au92.org>
|
||||
James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
|
||||
janost <janost@tuta.io>
|
||||
Jaroslav Lichtblau <svetlemodry@users.noreply.github.com>
|
||||
Jaroslav Malec (dzarda) <dzardacz@gmail.com>
|
||||
jaseg <githubaccount@jaseg.net>
|
||||
Jauder Ho <jauderho@users.noreply.github.com>
|
||||
Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
|
||||
Jaya Kumar <jaya.kumar@ict.nl>
|
||||
Jeffery To <jeffery.to@gmail.com>
|
||||
jelle van der Waa <jelle@vdwaa.nl>
|
||||
Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
|
||||
Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
|
||||
@@ -174,18 +140,15 @@ Johan Andersson <j@i19.se>
|
||||
Johan Vromans (sciurius) <jvromans@squirrel.nl>
|
||||
John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
|
||||
Jonas Thelemann <e-mail@jonas-thelemann.de>
|
||||
Jonathan <artback@protonmail.com> <jonagn@gmail.com>
|
||||
Jonathan <artback@protonmail.com>
|
||||
Jonathan Cross <jcross@gmail.com>
|
||||
Jonta <359397+Jonta@users.noreply.github.com>
|
||||
Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
|
||||
jtagcat <git-514635f7@jtag.cat> <git-12dbd862@jtag.cat>
|
||||
jtagcat <git-514635f7@jtag.cat>
|
||||
Jörg Thalheim <Mic92@users.noreply.github.com>
|
||||
Jędrzej Kula <kula.jedrek@gmail.com>
|
||||
K.B.Dharun Krishna <kbdharunkrishna@gmail.com>
|
||||
Kalle Laine <pahakalle@protonmail.com>
|
||||
Karol Różycki (krozycki) <rozycki.karol@gmail.com>
|
||||
Kebin Liu <lkebin@gmail.com>
|
||||
Keith Harrison <keithh@protonmail.com>
|
||||
Keith Turner <kturner@apache.org>
|
||||
Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
|
||||
Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
|
||||
@@ -202,9 +165,7 @@ Leo Arias (elopio) <yo@elopio.net>
|
||||
Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
|
||||
Lode Hoste (Zillode) <zillode@zillode.be>
|
||||
Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
|
||||
LSmithx2 <42276854+lsmithx2@users.noreply.github.com>
|
||||
Lukas Lihotzki <lukas@lihotzki.de>
|
||||
luzpaz <luzpaz@users.noreply.github.com>
|
||||
Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
|
||||
Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
|
||||
Marc Pujol (kilburn) <kilburn@la3.org>
|
||||
@@ -213,8 +174,6 @@ marco-m <marco.molteni@laposte.net>
|
||||
Marcus Legendre <marcus.legendre@gmail.com>
|
||||
Mario Majila <mariustshipichik@gmail.com>
|
||||
Mark Pulford (mpx) <mark@kyne.com.au>
|
||||
Martchus <martchus@gmx.net>
|
||||
Martin Polehla <p0l0us@users.noreply.github.com>
|
||||
Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
|
||||
Mateusz Ż <thedead4fun@live.com>
|
||||
Matic Potočnik <hairyfotr@gmail.com>
|
||||
@@ -222,27 +181,22 @@ Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
|
||||
Matt Robenolt <matt@ydekproductions.com>
|
||||
Matteo Ruina <matteo.ruina@gmail.com>
|
||||
Maurizio Tomasi <ziotom78@gmail.com>
|
||||
Max <github@germancoding.com>
|
||||
Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
|
||||
MaximAL <almaximal@ya.ru>
|
||||
Maxime Thirouin <m@moox.io>
|
||||
Maximilian <maxi.rostock@outlook.de> <public@complexvector.space>
|
||||
mclang <1721600+mclang@users.noreply.github.com>
|
||||
Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
|
||||
Michael Ploujnikov (plouj) <ploujj@gmail.com>
|
||||
Michael Rienstra <mrienstra@gmail.com>
|
||||
Michael Tilli (pyfisch) <pyfisch@gmail.com>
|
||||
MichaIng <micha@dietpi.com>
|
||||
Migelo <miha@filetki.si>
|
||||
Mike Boone <mike@boonedocks.net>
|
||||
MikeLund <MikeLund@users.noreply.github.com>
|
||||
MikolajTwarog <43782609+MikolajTwarog@users.noreply.github.com>
|
||||
Mingxuan Lin <gdlmx@users.noreply.github.com>
|
||||
mv1005 <49659413+mv1005@users.noreply.github.com>
|
||||
Nate Morrison (nrm21) <natemorrison@gmail.com>
|
||||
Naveen <172697+naveensrinivasan@users.noreply.github.com>
|
||||
Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
|
||||
Nick Busey <NickBusey@users.noreply.github.com>
|
||||
Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
|
||||
Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
|
||||
Nicolas Perraut <n.perraut@gmail.com>
|
||||
@@ -252,7 +206,6 @@ NinoM4ster <ninom4ster@gmail.com>
|
||||
Nitroretro <43112364+Nitroretro@users.noreply.github.com>
|
||||
NoLooseEnds <jon.koslung@gmail.com>
|
||||
Oliver Freyermuth <o.freyermuth@googlemail.com>
|
||||
orangekame3 <miya.org.0309@gmail.com>
|
||||
otbutz <tbutz@optitool.de>
|
||||
Otiel <Otiel@users.noreply.github.com>
|
||||
overkill <22098433+0verk1ll@users.noreply.github.com>
|
||||
@@ -276,7 +229,6 @@ Piotr Bejda (piobpl) <piotrb10@gmail.com>
|
||||
Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
|
||||
Quentin Hibon <qh.public@yahoo.com>
|
||||
Rahmi Pruitt <rjpruitt16@gmail.com>
|
||||
red_led <red-led@users.noreply.github.com>
|
||||
Richard Hartmann <RichiH@users.noreply.github.com>
|
||||
Robert Carosi (nov1n) <robert@carosi.nl>
|
||||
Roberto Santalla <roobre@users.noreply.github.com>
|
||||
@@ -285,14 +237,11 @@ Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
|
||||
Ross Smith II (rasa) <ross@smithii.com>
|
||||
rubenbe <github-com-00ff86@vandamme.email>
|
||||
Ruslan Yevdokymov <38809160+ruslanye@users.noreply.github.com>
|
||||
Ryan Qian <i@bitbili.net>
|
||||
Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
|
||||
Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
|
||||
Scott Klupfel (kluppy) <kluppy@going2blue.com>
|
||||
sec65 <106604020+sec65@users.noreply.github.com>
|
||||
Sergey Mishin (ralder) <ralder@yandex.ru>
|
||||
Sertonix <83883937+Sertonix@users.noreply.github.com>
|
||||
Shaarad Dalvi <60266155+shaaraddalvi@users.noreply.github.com> <shdalv@microsoft.com>
|
||||
Shaarad Dalvi <60266155+shaaraddalvi@users.noreply.github.com>
|
||||
Simon Frei (imsodin) <freisim93@gmail.com>
|
||||
Simon Mwepu <simonmwepu@gmail.com>
|
||||
Sly_tom_cat <slytomcat@mail.ru>
|
||||
@@ -300,8 +249,6 @@ Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
|
||||
Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
|
||||
Steven Eckhoff <steven.eckhoff.opensource@gmail.com>
|
||||
Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
|
||||
Syncthing Automation <automation@syncthing.net>
|
||||
Syncthing Release Automation <release@syncthing.net>
|
||||
Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
|
||||
Thomas Hipp <thomashipp@gmail.com>
|
||||
Tim Abell (timabell) <tim@timwise.co.uk>
|
||||
@@ -316,15 +263,11 @@ Tully Robinson (tojrobinson) <tully@tojr.org>
|
||||
Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
|
||||
Tyler Kropp <kropptyler@gmail.com>
|
||||
Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
|
||||
vapatel2 <149737089+vapatel2@users.noreply.github.com>
|
||||
Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
|
||||
Victor Buinsky (buinsky) <vix_booja@tut.by>
|
||||
Vik <63919734+ViktorOn@users.noreply.github.com>
|
||||
Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
|
||||
villekalliomaki <53118179+villekalliomaki@users.noreply.github.com>
|
||||
Vladimir Rusinov <vrusinov@google.com> <vladimir.rusinov@gmail.com>
|
||||
wangguoliang <liangcszzu@163.com>
|
||||
Will Rouesnel <wrouesnel@wrouesnel.com>
|
||||
William A. Kennington III (wkennington) <william@wkennington.com>
|
||||
wouter bolsterlee <wouter@bolsterl.ee>
|
||||
Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
|
||||
@@ -333,4 +276,3 @@ Xavier O. (damajor) <damajor@gmail.com>
|
||||
xjtdy888 (xjtdy888) <xjtdy888@163.com>
|
||||
Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
|
||||
佛跳墙 <daoquan@qq.com>
|
||||
落心 <luoxin.ttt@gmail.com>
|
||||
|
||||
@@ -24,15 +24,10 @@ too much information will never get you yelled at. :)
|
||||
## Contributing Translations
|
||||
|
||||
All translations are done via
|
||||
[Weblate](https://hosted.weblate.org/projects/syncthing/). If you wish
|
||||
to contribute to a translation, just head over there and sign up.
|
||||
[Transifex](https://www.transifex.com/projects/p/syncthing/). If you
|
||||
wish to contribute to a translation, just head over there and sign up.
|
||||
Before every release, the language resources are updated from the
|
||||
latest info on Weblate.
|
||||
|
||||
Note that the previously used service at
|
||||
[Transifex](https://www.transifex.com/projects/p/syncthing/) is being
|
||||
retired and we kindly ask you to sign up on Weblate for continued
|
||||
involvement.
|
||||
latest info on Transifex.
|
||||
|
||||
## Contributing Code
|
||||
|
||||
|
||||
42
Dockerfile
42
Dockerfile
@@ -1,57 +1,29 @@
|
||||
ARG GOVERSION=latest
|
||||
|
||||
#
|
||||
# Maybe build Syncthing. This is a bit ugly as we can't make an entire
|
||||
# section of the Dockerfile conditional, so we end up always pulling the
|
||||
# golang image as builder. Then we check if the executable we need already
|
||||
# exists (pre-built) otherwise we build it.
|
||||
#
|
||||
|
||||
FROM golang:$GOVERSION AS builder
|
||||
ARG BUILD_USER
|
||||
ARG BUILD_HOST
|
||||
ARG TARGETARCH
|
||||
|
||||
WORKDIR /src
|
||||
COPY . .
|
||||
|
||||
ENV CGO_ENABLED=0
|
||||
RUN if [ ! -f syncthing-linux-$TARGETARCH ] ; then \
|
||||
go run build.go -no-upgrade build syncthing ; \
|
||||
mv syncthing syncthing-linux-$TARGETARCH ; \
|
||||
fi
|
||||
|
||||
#
|
||||
# The rest of the Dockerfile uses the binary from the builder, prebuilt or
|
||||
# not.
|
||||
#
|
||||
ENV BUILD_HOST=syncthing.net
|
||||
ENV BUILD_USER=docker
|
||||
RUN rm -f syncthing && go run build.go -no-upgrade build syncthing
|
||||
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing"
|
||||
|
||||
EXPOSE 8384 22000/tcp 22000/udp 21027/udp
|
||||
|
||||
VOLUME ["/var/syncthing"]
|
||||
|
||||
RUN apk add --no-cache ca-certificates curl libcap su-exec tzdata
|
||||
RUN apk add --no-cache ca-certificates su-exec tzdata
|
||||
|
||||
COPY --from=builder /src/syncthing-linux-$TARGETARCH /bin/syncthing
|
||||
COPY --from=builder /src/syncthing /bin/syncthing
|
||||
COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
|
||||
|
||||
ENV PUID=1000 PGID=1000 HOME=/var/syncthing
|
||||
|
||||
HEALTHCHECK --interval=1m --timeout=10s \
|
||||
CMD curl -fkLsS -m 2 127.0.0.1:8384/rest/noauth/health | grep -o --color=never OK || exit 1
|
||||
CMD nc -z 127.0.0.1 8384 || exit 1
|
||||
|
||||
ENV STGUIADDRESS=0.0.0.0:8384
|
||||
ENV STHOMEDIR=/var/syncthing/config
|
||||
RUN chmod 755 /bin/entrypoint.sh
|
||||
ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing"]
|
||||
ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]
|
||||
|
||||
@@ -1,17 +1,9 @@
|
||||
ARG GOVERSION=latest
|
||||
FROM golang:$GOVERSION
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing Builder"
|
||||
|
||||
# FPM to build Debian packages
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
locales rubygems ruby-dev build-essential git \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& gem install fpm
|
||||
&& gem install --no-ri --no-rdoc fpm
|
||||
|
||||
19
Dockerfile.buildx
Normal file
19
Dockerfile.buildx
Normal file
@@ -0,0 +1,19 @@
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
EXPOSE 8384 22000/tcp 22000/udp 21027/udp
|
||||
|
||||
VOLUME ["/var/syncthing"]
|
||||
|
||||
RUN apk add --no-cache ca-certificates su-exec tzdata
|
||||
|
||||
COPY ./syncthing-linux-$TARGETARCH /bin/syncthing
|
||||
COPY ./script/docker-entrypoint.sh /bin/entrypoint.sh
|
||||
|
||||
ENV PUID=1000 PGID=1000 HOME=/var/syncthing
|
||||
|
||||
HEALTHCHECK --interval=1m --timeout=10s \
|
||||
CMD nc -z 127.0.0.1 8384 || exit 1
|
||||
|
||||
ENV STGUIADDRESS=0.0.0.0:8384
|
||||
ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]
|
||||
@@ -1,16 +0,0 @@
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing Crash Receiver"
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
COPY stcrashreceiver-linux-${TARGETARCH} /bin/stcrashreceiver
|
||||
|
||||
ENTRYPOINT [ "/bin/stcrashreceiver" ]
|
||||
@@ -1,28 +1,15 @@
|
||||
ARG GOVERSION=latest
|
||||
FROM golang:$GOVERSION AS builder
|
||||
ARG BUILD_USER
|
||||
ARG BUILD_HOST
|
||||
ARG TARGETARCH
|
||||
|
||||
WORKDIR /src
|
||||
COPY . .
|
||||
|
||||
ENV CGO_ENABLED=0
|
||||
RUN if [ ! -f stdiscosrv-linux-$TARGETARCH ] ; then \
|
||||
go run build.go -no-upgrade build stdiscosrv ; \
|
||||
mv stdiscosrv stdiscosrv-linux-$TARGETARCH ; \
|
||||
fi
|
||||
ENV BUILD_HOST=syncthing.net
|
||||
ENV BUILD_USER=docker
|
||||
RUN rm -f stdiscosrv && go run build.go -no-upgrade build stdiscosrv
|
||||
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing Discovery Server"
|
||||
|
||||
EXPOSE 19200 8443
|
||||
|
||||
@@ -30,7 +17,7 @@ VOLUME ["/var/stdiscosrv"]
|
||||
|
||||
RUN apk add --no-cache ca-certificates su-exec
|
||||
|
||||
COPY --from=builder /src/stdiscosrv-linux-$TARGETARCH /bin/stdiscosrv
|
||||
COPY --from=builder /src/stdiscosrv /bin/stdiscosrv
|
||||
COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
|
||||
|
||||
ENV PUID=1000 PGID=1000 HOME=/var/stdiscosrv
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing Relay Pool Server"
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
RUN apk add --no-cache ca-certificates su-exec curl
|
||||
ENV PUID=1000 PGID=1000 MAXMIND_KEY=
|
||||
|
||||
RUN mkdir /var/strelaypoolsrv && chown 1000 /var/strelaypoolsrv
|
||||
USER 1000
|
||||
|
||||
COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv
|
||||
COPY script/strelaypoolsrv-entrypoint.sh /bin/entrypoint.sh
|
||||
|
||||
WORKDIR /var/strelaypoolsrv
|
||||
ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaypoolsrv", "-listen", ":8080"]
|
||||
@@ -1,28 +1,15 @@
|
||||
ARG GOVERSION=latest
|
||||
FROM golang:$GOVERSION AS builder
|
||||
ARG BUILD_USER
|
||||
ARG BUILD_HOST
|
||||
ARG TARGETARCH
|
||||
|
||||
WORKDIR /src
|
||||
COPY . .
|
||||
|
||||
ENV CGO_ENABLED=0
|
||||
RUN if [ ! -f strelaysrv-linux-$TARGETARCH ] ; then \
|
||||
go run build.go -no-upgrade build strelaysrv ; \
|
||||
mv strelaysrv strelaysrv-linux-$TARGETARCH ; \
|
||||
fi
|
||||
ENV BUILD_HOST=syncthing.net
|
||||
ENV BUILD_USER=docker
|
||||
RUN rm -f strelaysrv && go run build.go -no-upgrade build strelaysrv
|
||||
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing Relay Server"
|
||||
|
||||
EXPOSE 22067 22070
|
||||
|
||||
@@ -30,7 +17,7 @@ VOLUME ["/var/strelaysrv"]
|
||||
|
||||
RUN apk add --no-cache ca-certificates su-exec
|
||||
|
||||
COPY --from=builder /src/strelaysrv-linux-$TARGETARCH /bin/strelaysrv
|
||||
COPY --from=builder /src/strelaysrv /bin/strelaysrv
|
||||
COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
|
||||
|
||||
ENV PUID=1000 PGID=1000 HOME=/var/strelaysrv
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing Upgrades"
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
COPY stupgrades-linux-${TARGETARCH} /bin/stupgrades
|
||||
|
||||
ENTRYPOINT [ "/bin/stupgrades" ]
|
||||
@@ -1,16 +0,0 @@
|
||||
FROM alpine
|
||||
ARG TARGETARCH
|
||||
|
||||
LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
org.opencontainers.image.url="https://syncthing.net" \
|
||||
org.opencontainers.image.documentation="https://docs.syncthing.net" \
|
||||
org.opencontainers.image.source="https://github.com/syncthing/syncthing" \
|
||||
org.opencontainers.image.vendor="The Syncthing Project" \
|
||||
org.opencontainers.image.licenses="MPL-2.0" \
|
||||
org.opencontainers.image.title="Syncthing Usage Reporting Server"
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
COPY ursrv-linux-${TARGETARCH} /bin/ursrv
|
||||
|
||||
ENTRYPOINT [ "/bin/ursrv" ]
|
||||
14
GOALS.md
14
GOALS.md
@@ -24,17 +24,17 @@ to avoid corrupting the user's files.
|
||||
### 2. Secure Against Attackers
|
||||
|
||||
Again, protecting the user's data is paramount. Regardless of our other
|
||||
goals, we must never allow the user's data to be susceptible to eavesdropping
|
||||
goals we must never allow the user's data to be susceptible to eavesdropping
|
||||
or modification by unauthorized parties.
|
||||
|
||||
> This should be understood in context. It is not necessarily reasonable to
|
||||
> expect Syncthing to be resistant against well equipped state level
|
||||
> attackers. We will, however, do our best. Note also that this is different
|
||||
> attackers. We will however do our best. Note also that this is different
|
||||
> from anonymity which is not, currently, a goal.
|
||||
|
||||
### 3. Easy to Use
|
||||
|
||||
Syncthing should be approachable, understandable, and inclusive.
|
||||
Syncthing should be approachable, understandable and inclusive.
|
||||
|
||||
> Complex concepts and maths form the base of Syncthing's functionality.
|
||||
> This should nonetheless be abstracted or hidden to a degree where
|
||||
@@ -52,18 +52,18 @@ User interaction should be required only when absolutely necessary.
|
||||
### 5. Universally Available
|
||||
|
||||
Syncthing should run on every common computer. We are mindful that the
|
||||
latest technology is not always available to every individual.
|
||||
latest technology is not always available to any given individual.
|
||||
|
||||
> Computers include desktops, laptops, servers, virtual machines, small
|
||||
> general purpose computers such as Raspberry Pis and, *where possible*,
|
||||
> tablets and phones. NAS appliances, toasters, cars, firearms, thermostats,
|
||||
> and so on may include computing capabilities but it is not our goal for
|
||||
> tablets and phones. NAS appliances, toasters, cars, firearms, thermostats
|
||||
> and so on may include computing capabitilies but it is not our goal for
|
||||
> Syncthing to run smoothly on these devices.
|
||||
|
||||
### 6. For Individuals
|
||||
|
||||
Syncthing is primarily about empowering the individual user with safe,
|
||||
secure, and easy to use file synchronization.
|
||||
secure and easy to use file synchronization.
|
||||
|
||||
> We acknowledge that it's also useful in an enterprise setting and include
|
||||
> functionality to support that. If this is in conflict with the
|
||||
|
||||
@@ -7,27 +7,23 @@ Use the `/var/syncthing` volume to have the synchronized files available on the
|
||||
host. You can add more folders and map them as you prefer.
|
||||
|
||||
Note that Syncthing runs as UID 1000 and GID 1000 by default. These may be
|
||||
altered with the `PUID` and `PGID` environment variables. In addition
|
||||
altered with the ``PUID`` and ``PGID`` environment variables. In addition
|
||||
the name of the Syncthing instance can be optionally defined by using
|
||||
`--hostname=syncthing` parameter.
|
||||
|
||||
To grant Syncthing additional capabilities without running as root, use the
|
||||
`PCAP` environment variable with the same syntax as that for `setcap(8)`.
|
||||
For example, `PCAP=cap_chown,cap_fowner+ep`.
|
||||
``--hostname=syncthing`` parameter.
|
||||
|
||||
## Example Usage
|
||||
|
||||
**Docker cli**
|
||||
```
|
||||
$ docker pull syncthing/syncthing
|
||||
$ docker run -p 8384:8384 -p 22000:22000/tcp -p 22000:22000/udp -p 21027:21027/udp \
|
||||
$ docker run -p 8384:8384 -p 22000:22000/tcp -p 22000:22000/udp \
|
||||
-v /wherever/st-sync:/var/syncthing \
|
||||
--hostname=my-syncthing \
|
||||
syncthing/syncthing:latest
|
||||
```
|
||||
|
||||
**Docker compose**
|
||||
```yml
|
||||
```
|
||||
---
|
||||
version: "3"
|
||||
services:
|
||||
@@ -41,23 +37,20 @@ services:
|
||||
volumes:
|
||||
- /wherever/st-sync:/var/syncthing
|
||||
ports:
|
||||
- 8384:8384 # Web UI
|
||||
- 22000:22000/tcp # TCP file transfers
|
||||
- 22000:22000/udp # QUIC file transfers
|
||||
- 21027:21027/udp # Receive local discovery broadcasts
|
||||
- 8384:8384
|
||||
- 22000:22000/tcp
|
||||
- 22000:22000/udp
|
||||
restart: unless-stopped
|
||||
```
|
||||
|
||||
## Discovery
|
||||
|
||||
Note that Docker's default network mode prevents local IP addresses from
|
||||
being discovered, as Syncthing is only able to see the internal IP of the
|
||||
container on the `172.17.0.0/16` subnet. This will result in poor transfer rates
|
||||
if local device addresses are not manually configured.
|
||||
Note that local device discovery will not work with the above command,
|
||||
resulting in poor local transfer rates if local device addresses are not
|
||||
manually configured.
|
||||
|
||||
It is therefore advisable to use the [host network mode](https://docs.docker.com/network/host/) instead:
|
||||
To allow local discovery, the docker host network can be used instead:
|
||||
|
||||
**Docker cli**
|
||||
```
|
||||
$ docker pull syncthing/syncthing
|
||||
$ docker run --network=host \
|
||||
@@ -65,24 +58,6 @@ $ docker run --network=host \
|
||||
syncthing/syncthing:latest
|
||||
```
|
||||
|
||||
**Docker compose**
|
||||
```yml
|
||||
---
|
||||
version: "3"
|
||||
services:
|
||||
syncthing:
|
||||
image: syncthing/syncthing
|
||||
container_name: syncthing
|
||||
hostname: my-syncthing
|
||||
environment:
|
||||
- PUID=1000
|
||||
- PGID=1000
|
||||
volumes:
|
||||
- /wherever/st-sync:/var/syncthing
|
||||
network_mode: host
|
||||
restart: unless-stopped
|
||||
```
|
||||
|
||||
Be aware that syncthing alone is now in control of what interfaces and ports it
|
||||
listens on. You can edit the syncthing configuration to change the defaults if
|
||||
there are conflicts.
|
||||
|
||||
38
README.md
38
README.md
@@ -2,6 +2,9 @@
|
||||
|
||||
---
|
||||
|
||||
[](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildLinuxCross&guest=1)
|
||||
[](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildWindows&guest=1)
|
||||
[](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildMac&guest=1)
|
||||
[](https://www.mozilla.org/MPL/2.0/)
|
||||
[](https://bestpractices.coreinfrastructure.org/projects/88)
|
||||
[](https://goreportcard.com/report/github.com/syncthing/syncthing)
|
||||
@@ -10,8 +13,8 @@
|
||||
|
||||
Syncthing is a **continuous file synchronization program**. It synchronizes
|
||||
files between two or more computers. We strive to fulfill the goals below.
|
||||
The goals are listed in order of importance, the most important ones first.
|
||||
This is the summary version of the goal list - for more
|
||||
The goals are listed in order of importance, the most important one being
|
||||
the first. This is the summary version of the goal list - for more
|
||||
commentary, see the full [Goals document][13].
|
||||
|
||||
Syncthing should be:
|
||||
@@ -24,12 +27,12 @@ Syncthing should be:
|
||||
2. **Secure Against Attackers**
|
||||
|
||||
Again, protecting the user's data is paramount. Regardless of our other
|
||||
goals, we must never allow the user's data to be susceptible to
|
||||
goals we must never allow the user's data to be susceptible to
|
||||
eavesdropping or modification by unauthorized parties.
|
||||
|
||||
3. **Easy to Use**
|
||||
|
||||
Syncthing should be approachable, understandable, and inclusive.
|
||||
Syncthing should be approachable, understandable and inclusive.
|
||||
|
||||
4. **Automatic**
|
||||
|
||||
@@ -38,12 +41,12 @@ Syncthing should be:
|
||||
5. **Universally Available**
|
||||
|
||||
Syncthing should run on every common computer. We are mindful that the
|
||||
latest technology is not always available to every individual.
|
||||
latest technology is not always available to any given individual.
|
||||
|
||||
6. **For Individuals**
|
||||
|
||||
Syncthing is primarily about empowering the individual user with safe,
|
||||
secure, and easy to use file synchronization.
|
||||
secure and easy to use file synchronization.
|
||||
|
||||
7. **Everything Else**
|
||||
|
||||
@@ -57,7 +60,7 @@ Take a look at the [getting started guide][2].
|
||||
|
||||
There are a few examples for keeping Syncthing running in the background
|
||||
on your system in [the etc directory][3]. There are also several [GUI
|
||||
implementations][11] for Windows, Mac, and Linux.
|
||||
implementations][11] for Windows, Mac and Linux.
|
||||
|
||||
## Docker
|
||||
|
||||
@@ -66,8 +69,7 @@ To run Syncthing in Docker, see [the Docker README][16].
|
||||
## Vote on features/bugs
|
||||
|
||||
We'd like to encourage you to [vote][12] on issues that matter to you.
|
||||
This helps the team understand what are the biggest pain points for our
|
||||
users, and could potentially influence what is being worked on next.
|
||||
This helps the team understand what are the biggest pain points for our users, and could potentially influence what is being worked on next.
|
||||
|
||||
## Getting in Touch
|
||||
|
||||
@@ -75,30 +77,24 @@ The first and best point of contact is the [Forum][8].
|
||||
If you've found something that is clearly a
|
||||
bug, feel free to report it in the [GitHub issue tracker][10].
|
||||
|
||||
If you believe that you’ve found a Syncthing-related security vulnerability,
|
||||
please report it by emailing security@syncthing.net. Do not report it in the
|
||||
Forum or issue tracker.
|
||||
|
||||
## Building
|
||||
|
||||
Building Syncthing from source is easy. After extracting the source bundle from
|
||||
a release or checking out git, you just need to run `go run build.go` and the
|
||||
binaries are created in `./bin`. There's [a guide][5] with more details on the
|
||||
build process.
|
||||
Building Syncthing from source is easy, and there's [a guide][5]
|
||||
that describes it for both Unix and Windows systems.
|
||||
|
||||
## Signed Releases
|
||||
|
||||
As of v0.10.15 and onwards, release binaries are GPG signed with the key
|
||||
As of v0.10.15 and onwards release binaries are GPG signed with the key
|
||||
D26E6ED000654A3E, available from https://syncthing.net/security.html and
|
||||
most key servers.
|
||||
|
||||
There is also a built-in automatic upgrade mechanism (disabled in some
|
||||
There is also a built in automatic upgrade mechanism (disabled in some
|
||||
distribution channels) which uses a compiled in ECDSA signature. macOS
|
||||
binaries are also properly code signed.
|
||||
|
||||
## Documentation
|
||||
|
||||
Please see the Syncthing [documentation site][6] [[source]][17].
|
||||
Please see the [Syncthing documentation site][6].
|
||||
|
||||
All code is licensed under the [MPLv2 License][7].
|
||||
|
||||
@@ -116,4 +112,4 @@ All code is licensed under the [MPLv2 License][7].
|
||||
[14]: assets/logo-text-128.png
|
||||
[15]: https://syncthing.net/
|
||||
[16]: https://github.com/syncthing/syncthing/blob/main/README-Docker.md
|
||||
[17]: https://github.com/syncthing/docs
|
||||
|
||||
|
||||
126
build.go
126
build.go
@@ -4,7 +4,6 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
//go:build ignore
|
||||
// +build ignore
|
||||
|
||||
package main
|
||||
@@ -15,12 +14,12 @@ import (
|
||||
"bytes"
|
||||
"compress/flate"
|
||||
"compress/gzip"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -32,8 +31,6 @@ import (
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
buildpkg "github.com/syncthing/syncthing/lib/build"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -50,7 +47,6 @@ var (
|
||||
cc string
|
||||
run string
|
||||
benchRun string
|
||||
buildOut string
|
||||
debugBinary bool
|
||||
coverage bool
|
||||
long bool
|
||||
@@ -146,7 +142,7 @@ var targets = map[string]target{
|
||||
{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
|
||||
{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
|
||||
},
|
||||
systemdService: "stdiscosrv.service",
|
||||
systemdService: "cmd/stdiscosrv/etc/linux-systemd/stdiscosrv.service",
|
||||
installationFiles: []archiveFile{
|
||||
{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
|
||||
{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0644},
|
||||
@@ -174,7 +170,7 @@ var targets = map[string]target{
|
||||
{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
|
||||
{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
|
||||
},
|
||||
systemdService: "strelaysrv.service",
|
||||
systemdService: "cmd/strelaysrv/etc/linux-systemd/strelaysrv.service",
|
||||
installationFiles: []archiveFile{
|
||||
{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
|
||||
{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0644},
|
||||
@@ -207,24 +203,6 @@ var targets = map[string]target{
|
||||
{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaypoolsrv/AUTHORS.txt", perm: 0644},
|
||||
},
|
||||
},
|
||||
"stupgrades": {
|
||||
name: "stupgrades",
|
||||
description: "Syncthing Upgrade Check Server",
|
||||
buildPkgs: []string{"github.com/syncthing/syncthing/cmd/stupgrades"},
|
||||
binaryName: "stupgrades",
|
||||
},
|
||||
"stcrashreceiver": {
|
||||
name: "stcrashreceiver",
|
||||
description: "Syncthing Crash Server",
|
||||
buildPkgs: []string{"github.com/syncthing/syncthing/cmd/stcrashreceiver"},
|
||||
binaryName: "stcrashreceiver",
|
||||
},
|
||||
"ursrv": {
|
||||
name: "ursrv",
|
||||
description: "Syncthing Usage Reporting Server",
|
||||
buildPkgs: []string{"github.com/syncthing/syncthing/cmd/ursrv"},
|
||||
binaryName: "ursrv",
|
||||
},
|
||||
}
|
||||
|
||||
func initTargets() {
|
||||
@@ -323,9 +301,6 @@ func runCommand(cmd string, target target) {
|
||||
case "assets":
|
||||
rebuildAssets()
|
||||
|
||||
case "update-deps":
|
||||
updateDependencies()
|
||||
|
||||
case "proto":
|
||||
proto()
|
||||
|
||||
@@ -338,9 +313,6 @@ func runCommand(cmd string, target target) {
|
||||
case "transifex":
|
||||
transifex()
|
||||
|
||||
case "weblate":
|
||||
weblate()
|
||||
|
||||
case "tar":
|
||||
buildTar(target, tags)
|
||||
|
||||
@@ -399,7 +371,6 @@ func parseFlags() {
|
||||
flag.StringVar(&run, "run", "", "Specify which tests to run")
|
||||
flag.StringVar(&benchRun, "bench", "", "Specify which benchmarks to run")
|
||||
flag.BoolVar(&withNextGenGUI, "with-next-gen-gui", withNextGenGUI, "Also build 'newgui'")
|
||||
flag.StringVar(&buildOut, "build-out", "", "Set the '-o' value for 'go build'")
|
||||
flag.Parse()
|
||||
}
|
||||
|
||||
@@ -417,7 +388,7 @@ func test(tags []string, pkgs ...string) {
|
||||
|
||||
if runtime.GOARCH == "amd64" {
|
||||
switch runtime.GOOS {
|
||||
case buildpkg.Darwin, buildpkg.Linux, buildpkg.FreeBSD: // , "windows": # See https://github.com/golang/go/issues/27089
|
||||
case "darwin", "linux", "freebsd": // , "windows": # See https://github.com/golang/go/issues/27089
|
||||
args = append(args, "-race")
|
||||
}
|
||||
}
|
||||
@@ -532,9 +503,6 @@ func build(target target, tags []string) {
|
||||
}
|
||||
|
||||
args := []string{"build", "-v"}
|
||||
if buildOut != "" {
|
||||
args = append(args, "-o", buildOut)
|
||||
}
|
||||
args = appendParameters(args, tags, target.buildPkgs...)
|
||||
runPrint(goCmd, args...)
|
||||
}
|
||||
@@ -751,7 +719,7 @@ func shouldBuildSyso(dir string) (string, error) {
|
||||
}
|
||||
|
||||
jsonPath := filepath.Join(dir, "versioninfo.json")
|
||||
err = os.WriteFile(jsonPath, bs, 0644)
|
||||
err = ioutil.WriteFile(jsonPath, bs, 0644)
|
||||
if err != nil {
|
||||
return "", errors.New("failed to create " + jsonPath + ": " + err.Error())
|
||||
}
|
||||
@@ -764,13 +732,7 @@ func shouldBuildSyso(dir string) (string, error) {
|
||||
|
||||
sysoPath := filepath.Join(dir, "cmd", "syncthing", "resource.syso")
|
||||
|
||||
// See https://github.com/josephspurrier/goversioninfo#command-line-flags
|
||||
armOption := ""
|
||||
if strings.Contains(goarch, "arm") {
|
||||
armOption = "-arm=true"
|
||||
}
|
||||
|
||||
if _, err := runError("goversioninfo", "-o", sysoPath, armOption); err != nil {
|
||||
if _, err := runError("goversioninfo", "-o", sysoPath); err != nil {
|
||||
return "", errors.New("failed to create " + sysoPath + ": " + err.Error())
|
||||
}
|
||||
|
||||
@@ -790,12 +752,12 @@ func shouldCleanupSyso(sysoFilePath string) {
|
||||
// exists. The permission bits are copied as well. If dst already exists and
|
||||
// the contents are identical to src the modification time is not updated.
|
||||
func copyFile(src, dst string, perm os.FileMode) error {
|
||||
in, err := os.ReadFile(src)
|
||||
in, err := ioutil.ReadFile(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
out, err := os.ReadFile(dst)
|
||||
out, err := ioutil.ReadFile(dst)
|
||||
if err != nil {
|
||||
// The destination probably doesn't exist, we should create
|
||||
// it.
|
||||
@@ -811,7 +773,7 @@ func copyFile(src, dst string, perm os.FileMode) error {
|
||||
|
||||
copy:
|
||||
os.MkdirAll(filepath.Dir(dst), 0777)
|
||||
if err := os.WriteFile(dst, in, perm); err != nil {
|
||||
if err := ioutil.WriteFile(dst, in, perm); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -905,32 +867,12 @@ func shouldRebuildAssets(target, srcdir string) bool {
|
||||
return assetsAreNewer
|
||||
}
|
||||
|
||||
func updateDependencies() {
|
||||
// Figure out desired Go version
|
||||
bs, err := os.ReadFile("go.mod")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
re := regexp.MustCompile(`(?m)^go\s+([0-9.]+)`)
|
||||
matches := re.FindSubmatch(bs)
|
||||
if len(matches) != 2 {
|
||||
log.Fatal("failed to parse go.mod")
|
||||
}
|
||||
goVersion := string(matches[1])
|
||||
|
||||
runPrint(goCmd, "get", "-u", "./...")
|
||||
runPrint(goCmd, "mod", "tidy", "-go="+goVersion, "-compat="+goVersion)
|
||||
|
||||
// We might have updated the protobuf package and should regenerate to match.
|
||||
proto()
|
||||
}
|
||||
|
||||
func proto() {
|
||||
pv := protobufVersion()
|
||||
repo := "https://github.com/gogo/protobuf.git"
|
||||
path := filepath.Join("repos", "protobuf")
|
||||
|
||||
runPrint(goCmd, "install", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
|
||||
runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
|
||||
os.MkdirAll("repos", 0755)
|
||||
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
@@ -974,11 +916,6 @@ func transifex() {
|
||||
runPrint(goCmd, "run", "../../../../script/transifexdl.go")
|
||||
}
|
||||
|
||||
func weblate() {
|
||||
os.Chdir("gui/default/assets/lang")
|
||||
runPrint(goCmd, "run", "../../../../script/weblatedl.go")
|
||||
}
|
||||
|
||||
func ldflags(tags []string) string {
|
||||
b := new(strings.Builder)
|
||||
b.WriteString("-w")
|
||||
@@ -1003,7 +940,7 @@ func rmr(paths ...string) {
|
||||
}
|
||||
|
||||
func getReleaseVersion() (string, error) {
|
||||
bs, err := os.ReadFile("RELEASE")
|
||||
bs, err := ioutil.ReadFile("RELEASE")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -1026,7 +963,7 @@ func getGitVersion() (string, error) {
|
||||
v0 := string(bs)
|
||||
|
||||
// To be more semantic-versionish and ensure proper ordering in our
|
||||
// upgrade process, we make sure there's only one hyphen in the version.
|
||||
// upgrade process, we make sure there's only one hypen in the version.
|
||||
|
||||
versionRe := regexp.MustCompile(`-([0-9]{1,3}-g[0-9a-f]{5,10}(-dirty)?)`)
|
||||
if m := versionRe.FindStringSubmatch(vcur); len(m) > 0 {
|
||||
@@ -1114,14 +1051,10 @@ func getBranchSuffix() string {
|
||||
|
||||
branch = parts[len(parts)-1]
|
||||
switch branch {
|
||||
case "release", "main":
|
||||
case "master", "release", "main":
|
||||
// these are not special
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(branch, "release-") {
|
||||
// release branches are not special
|
||||
return ""
|
||||
}
|
||||
|
||||
validBranchRe := regexp.MustCompile(`^[a-zA-Z0-9_.-]+$`)
|
||||
if !validBranchRe.MatchString(branch) {
|
||||
@@ -1339,11 +1272,11 @@ func zipFile(out string, files []archiveFile) {
|
||||
|
||||
if strings.HasSuffix(f.dst, ".txt") {
|
||||
// Text file. Read it and convert line endings.
|
||||
bs, err := io.ReadAll(sf)
|
||||
bs, err := ioutil.ReadAll(sf)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\r', '\n'}, -1)
|
||||
bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\n', '\r'}, -1)
|
||||
fh.UncompressedSize = uint32(len(bs))
|
||||
fh.UncompressedSize64 = uint64(len(bs))
|
||||
|
||||
@@ -1414,33 +1347,6 @@ func windowsCodesign(file string) {
|
||||
args := []string{"sign", "/fd", algo}
|
||||
if f := os.Getenv("CODESIGN_CERTIFICATE_FILE"); f != "" {
|
||||
args = append(args, "/f", f)
|
||||
} else if b := os.Getenv("CODESIGN_CERTIFICATE_BASE64"); b != "" {
|
||||
// Decode the PFX certificate from base64.
|
||||
bs, err := base64.RawStdEncoding.DecodeString(b)
|
||||
if err != nil {
|
||||
log.Println("Codesign: signing failed: decoding base64:", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Write it to a temporary file
|
||||
f, err := os.CreateTemp("", "codesign-*.pfx")
|
||||
if err != nil {
|
||||
log.Println("Codesign: signing failed: creating temp file:", err)
|
||||
return
|
||||
}
|
||||
_ = f.Chmod(0600) // best effort remove other users' access
|
||||
defer os.Remove(f.Name())
|
||||
if _, err := f.Write(bs); err != nil {
|
||||
log.Println("Codesign: signing failed: writing temp file:", err)
|
||||
return
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
log.Println("Codesign: signing failed: closing temp file:", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Use that when signing
|
||||
args = append(args, "/f", f.Name())
|
||||
}
|
||||
if p := os.Getenv("CODESIGN_CERTIFICATE_PASSWORD"); p != "" {
|
||||
args = append(args, "/p", p)
|
||||
@@ -1460,7 +1366,7 @@ func windowsCodesign(file string) {
|
||||
|
||||
bs, err := runError(st, args...)
|
||||
if err != nil {
|
||||
log.Printf("Codesign: signing failed: %v: %s", err, string(bs))
|
||||
log.Println("Codesign: signing failed:", string(bs))
|
||||
return
|
||||
}
|
||||
log.Println("Codesign: successfully signed", file, "using", algo)
|
||||
|
||||
2
build.sh
2
build.sh
@@ -23,7 +23,7 @@ case "${1:-default}" in
|
||||
|
||||
prerelease)
|
||||
script authors
|
||||
build weblate
|
||||
build transifex
|
||||
pushd man ; ./refresh.sh ; popd
|
||||
git add -A gui man AUTHORS
|
||||
git commit -m 'gui, man, authors: Update docs, translations, and contributors'
|
||||
|
||||
@@ -1,198 +0,0 @@
|
||||
// Copyright (C) 2023 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
type diskStore struct {
|
||||
dir string
|
||||
inbox chan diskEntry
|
||||
maxBytes int64
|
||||
maxFiles int
|
||||
|
||||
currentFiles []currentFile
|
||||
currentSize int64
|
||||
}
|
||||
|
||||
type diskEntry struct {
|
||||
path string
|
||||
data []byte
|
||||
}
|
||||
|
||||
type currentFile struct {
|
||||
path string
|
||||
size int64
|
||||
mtime int64
|
||||
}
|
||||
|
||||
func (d *diskStore) Serve(ctx context.Context) {
|
||||
if err := os.MkdirAll(d.dir, 0o700); err != nil {
|
||||
log.Println("Creating directory:", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := d.inventory(); err != nil {
|
||||
log.Println("Failed to inventory disk store:", err)
|
||||
}
|
||||
d.clean()
|
||||
|
||||
cleanTimer := time.NewTicker(time.Minute)
|
||||
inventoryTimer := time.NewTicker(24 * time.Hour)
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
gw := gzip.NewWriter(buf)
|
||||
for {
|
||||
select {
|
||||
case entry := <-d.inbox:
|
||||
path := d.fullPath(entry.path)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
log.Println("Creating directory:", err)
|
||||
continue
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
gw.Reset(buf)
|
||||
if _, err := gw.Write(entry.data); err != nil {
|
||||
log.Println("Failed to compress crash report:", err)
|
||||
continue
|
||||
}
|
||||
if err := gw.Close(); err != nil {
|
||||
log.Println("Failed to compress crash report:", err)
|
||||
continue
|
||||
}
|
||||
if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
|
||||
log.Printf("Failed to write %s: %v", entry.path, err)
|
||||
_ = os.Remove(path)
|
||||
continue
|
||||
}
|
||||
|
||||
d.currentSize += int64(buf.Len())
|
||||
d.currentFiles = append(d.currentFiles, currentFile{
|
||||
size: int64(len(entry.data)),
|
||||
path: path,
|
||||
})
|
||||
|
||||
case <-cleanTimer.C:
|
||||
d.clean()
|
||||
|
||||
case <-inventoryTimer.C:
|
||||
if err := d.inventory(); err != nil {
|
||||
log.Println("Failed to inventory disk store:", err)
|
||||
}
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *diskStore) Put(path string, data []byte) bool {
|
||||
select {
|
||||
case d.inbox <- diskEntry{
|
||||
path: path,
|
||||
data: data,
|
||||
}:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (d *diskStore) Get(path string) ([]byte, error) {
|
||||
path = d.fullPath(path)
|
||||
bs, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gr, err := gzip.NewReader(bytes.NewReader(bs))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer gr.Close()
|
||||
return io.ReadAll(gr)
|
||||
}
|
||||
|
||||
func (d *diskStore) Exists(path string) bool {
|
||||
path = d.fullPath(path)
|
||||
_, err := os.Lstat(path)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (d *diskStore) clean() {
|
||||
for len(d.currentFiles) > 0 && (len(d.currentFiles) > d.maxFiles || d.currentSize > d.maxBytes) {
|
||||
f := d.currentFiles[0]
|
||||
log.Println("Removing", f.path)
|
||||
if err := os.Remove(f.path); err != nil {
|
||||
log.Println("Failed to remove file:", err)
|
||||
}
|
||||
d.currentFiles = d.currentFiles[1:]
|
||||
d.currentSize -= f.size
|
||||
}
|
||||
var oldest time.Duration
|
||||
if len(d.currentFiles) > 0 {
|
||||
oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
|
||||
}
|
||||
|
||||
metricDiskstoreFilesTotal.Set(float64(len(d.currentFiles)))
|
||||
metricDiskstoreBytesTotal.Set(float64(d.currentSize))
|
||||
metricDiskstoreOldestAgeSeconds.Set(math.Round(oldest.Seconds()))
|
||||
|
||||
log.Printf("Clean complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
|
||||
}
|
||||
|
||||
func (d *diskStore) inventory() error {
|
||||
d.currentFiles = nil
|
||||
d.currentSize = 0
|
||||
err := filepath.Walk(d.dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if filepath.Ext(path) != ".gz" {
|
||||
return nil
|
||||
}
|
||||
d.currentSize += info.Size()
|
||||
d.currentFiles = append(d.currentFiles, currentFile{
|
||||
path: path,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime().Unix(),
|
||||
})
|
||||
return nil
|
||||
})
|
||||
sort.Slice(d.currentFiles, func(i, j int) bool {
|
||||
return d.currentFiles[i].mtime < d.currentFiles[j].mtime
|
||||
})
|
||||
var oldest time.Duration
|
||||
if len(d.currentFiles) > 0 {
|
||||
oldest = time.Since(time.Unix(d.currentFiles[0].mtime, 0)).Truncate(time.Minute)
|
||||
}
|
||||
|
||||
metricDiskstoreFilesTotal.Set(float64(len(d.currentFiles)))
|
||||
metricDiskstoreBytesTotal.Set(float64(d.currentSize))
|
||||
metricDiskstoreOldestAgeSeconds.Set(math.Round(oldest.Seconds()))
|
||||
|
||||
log.Printf("Inventory complete: %d files, %d MB, oldest is %v ago", len(d.currentFiles), d.currentSize>>20, oldest)
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *diskStore) fullPath(path string) string {
|
||||
return filepath.Join(d.dir, path[0:2], path[2:]) + ".gz"
|
||||
}
|
||||
@@ -13,17 +13,17 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/syncthing/syncthing/lib/sha256"
|
||||
"github.com/syncthing/syncthing/lib/ur"
|
||||
|
||||
@@ -32,66 +32,34 @@ import (
|
||||
|
||||
const maxRequestSize = 1 << 20 // 1 MiB
|
||||
|
||||
type cli struct {
|
||||
Dir string `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."`
|
||||
DSN string `help:"Sentry DSN" env:"SENTRY_DSN"`
|
||||
Listen string `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"`
|
||||
MaxDiskFiles int `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"`
|
||||
MaxDiskSizeMB int64 `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"`
|
||||
SentryQueue int `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"`
|
||||
DiskQueue int `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
var params cli
|
||||
kong.Parse(¶ms)
|
||||
dir := flag.String("dir", ".", "Directory to store reports in")
|
||||
dsn := flag.String("dsn", "", "Sentry DSN")
|
||||
listen := flag.String("listen", ":22039", "HTTP listen address")
|
||||
flag.Parse()
|
||||
|
||||
mux := http.NewServeMux()
|
||||
|
||||
ds := &diskStore{
|
||||
dir: filepath.Join(params.Dir, "crash_reports"),
|
||||
inbox: make(chan diskEntry, params.DiskQueue),
|
||||
maxFiles: params.MaxDiskFiles,
|
||||
maxBytes: params.MaxDiskSizeMB << 20,
|
||||
}
|
||||
go ds.Serve(context.Background())
|
||||
|
||||
ss := &sentryService{
|
||||
dsn: params.DSN,
|
||||
inbox: make(chan sentryRequest, params.SentryQueue),
|
||||
}
|
||||
go ss.Serve(context.Background())
|
||||
|
||||
cr := &crashReceiver{
|
||||
store: ds,
|
||||
sentry: ss,
|
||||
dir: *dir,
|
||||
dsn: *dsn,
|
||||
}
|
||||
|
||||
mux.Handle("/", cr)
|
||||
mux.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) {
|
||||
w.Write([]byte("OK"))
|
||||
})
|
||||
mux.Handle("/metrics", promhttp.Handler())
|
||||
|
||||
if params.DSN != "" {
|
||||
mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports")))
|
||||
if *dsn != "" {
|
||||
mux.HandleFunc("/newcrash/failure", handleFailureFn(*dsn))
|
||||
}
|
||||
|
||||
log.SetOutput(os.Stdout)
|
||||
if err := http.ListenAndServe(params.Listen, mux); err != nil {
|
||||
if err := http.ListenAndServe(*listen, mux); err != nil {
|
||||
log.Fatalln("HTTP serve:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *http.Request) {
|
||||
func handleFailureFn(dsn string) func(w http.ResponseWriter, req *http.Request) {
|
||||
return func(w http.ResponseWriter, req *http.Request) {
|
||||
result := "failure"
|
||||
defer func() {
|
||||
metricFailureReportsTotal.WithLabelValues(result).Inc()
|
||||
}()
|
||||
|
||||
lr := io.LimitReader(req.Body, maxRequestSize)
|
||||
bs, err := io.ReadAll(lr)
|
||||
bs, err := ioutil.ReadAll(lr)
|
||||
req.Body.Close()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 500)
|
||||
@@ -121,18 +89,6 @@ func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *ht
|
||||
pkt.Extra = raven.Extra{
|
||||
"count": r.Count,
|
||||
}
|
||||
for k, v := range r.Extra {
|
||||
pkt.Extra[k] = v
|
||||
}
|
||||
if r.Goroutines != "" {
|
||||
url, err := saveFailureWithGoroutines(r.FailureData, failureDir)
|
||||
if err != nil {
|
||||
log.Println("Saving failure report:", err)
|
||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
pkt.Extra["goroutinesURL"] = url
|
||||
}
|
||||
message := sanitizeMessageLDB(r.Description)
|
||||
pkt.Fingerprint = []string{message}
|
||||
|
||||
@@ -140,21 +96,24 @@ func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *ht
|
||||
log.Println("Failed to send failure report:", err)
|
||||
} else {
|
||||
log.Println("Sent failure report:", r.Description)
|
||||
result = "success"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string, error) {
|
||||
bs := make([]byte, len(data.Description)+len(data.Goroutines))
|
||||
copy(bs, data.Description)
|
||||
copy(bs[len(data.Description):], data.Goroutines)
|
||||
id := fmt.Sprintf("%x", sha256.Sum256(bs))
|
||||
path := fullPathCompressed(failureDir, id)
|
||||
err := compressAndWrite(bs, path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
// userIDFor returns a string we can use as the user ID for the purpose of
|
||||
// counting affected users. It's the truncated hash of a salt, the user
|
||||
// remote IP, and the current month.
|
||||
func userIDFor(req *http.Request) string {
|
||||
addr := req.RemoteAddr
|
||||
if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
|
||||
addr = fwd
|
||||
}
|
||||
return reportServer + path, nil
|
||||
if host, _, err := net.SplitHostPort(addr); err == nil {
|
||||
addr = host
|
||||
}
|
||||
now := time.Now().Format("200601")
|
||||
salt := "stcrashreporter"
|
||||
hash := sha256.Sum256([]byte(salt + addr + now))
|
||||
return fmt.Sprintf("%x", hash[:8])
|
||||
}
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
// Copyright (C) 2023 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var (
|
||||
metricCrashReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "crash_reports_total",
|
||||
}, []string{"result"})
|
||||
metricFailureReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "failure_reports_total",
|
||||
}, []string{"result"})
|
||||
metricDiskstoreFilesTotal = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "diskstore_files_total",
|
||||
})
|
||||
metricDiskstoreBytesTotal = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "diskstore_bytes_total",
|
||||
})
|
||||
metricDiskstoreOldestAgeSeconds = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "crashreceiver",
|
||||
Name: "diskstore_oldest_age_seconds",
|
||||
})
|
||||
)
|
||||
@@ -8,16 +8,14 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"io/ioutil"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
raven "github.com/getsentry/raven-go"
|
||||
"github.com/maruel/panicparse/v2/stack"
|
||||
"github.com/maruel/panicparse/stack"
|
||||
)
|
||||
|
||||
const reportServer = "https://crash.syncthing.net/report/"
|
||||
@@ -33,45 +31,6 @@ var (
|
||||
clientsMut sync.Mutex
|
||||
)
|
||||
|
||||
type sentryService struct {
|
||||
dsn string
|
||||
inbox chan sentryRequest
|
||||
}
|
||||
|
||||
type sentryRequest struct {
|
||||
reportID string
|
||||
userID string
|
||||
data []byte
|
||||
}
|
||||
|
||||
func (s *sentryService) Serve(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case req := <-s.inbox:
|
||||
pkt, err := parseCrashReport(req.reportID, req.data)
|
||||
if err != nil {
|
||||
log.Println("Failed to parse crash report:", err)
|
||||
continue
|
||||
}
|
||||
if err := sendReport(s.dsn, pkt, req.userID); err != nil {
|
||||
log.Println("Failed to send crash report:", err)
|
||||
}
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *sentryService) Send(reportID, userID string, data []byte) bool {
|
||||
select {
|
||||
case s.inbox <- sentryRequest{reportID, userID, data}:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func sendReport(dsn string, pkt *raven.Packet, userID string) error {
|
||||
pkt.Interfaces = append(pkt.Interfaces, &raven.User{ID: userID})
|
||||
|
||||
@@ -134,13 +93,10 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
|
||||
}
|
||||
|
||||
r := bytes.NewReader(report)
|
||||
ctx, _, err := stack.ScanSnapshot(r, io.Discard, stack.DefaultOpts())
|
||||
if err != nil && err != io.EOF {
|
||||
ctx, err := stack.ParseDump(r, ioutil.Discard, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ctx == nil || len(ctx.Goroutines) == 0 {
|
||||
return nil, errors.New("no goroutines found")
|
||||
}
|
||||
|
||||
// Lock the source code loader to the version we are processing here.
|
||||
if version.commit != "" {
|
||||
@@ -160,7 +116,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) {
|
||||
if gr.First {
|
||||
trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
|
||||
for i, sc := range gr.Stack.Calls {
|
||||
trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name, sc.RemoteSrcPath, sc.Line, 3, nil)
|
||||
trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name(), sc.SrcPath, sc.Line, 3, nil)
|
||||
}
|
||||
break
|
||||
}
|
||||
@@ -216,13 +172,7 @@ func crashReportFingerprint(message string) []string {
|
||||
}
|
||||
|
||||
// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]
|
||||
// or, somewhere along the way the "+" in the version tag disappeared:
|
||||
// syncthing v1.23.7-dev.26.gdf7b56ae.dirty-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]
|
||||
var (
|
||||
longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
|
||||
gitExtraRE = regexp.MustCompile(`\.\d+\.g[0-9a-f]+`) // ".1.g6aaae618"
|
||||
gitExtraSepRE = regexp.MustCompile(`[.-]`) // dot or dash
|
||||
)
|
||||
var longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`)
|
||||
|
||||
type version struct {
|
||||
version string // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
|
||||
@@ -264,21 +214,10 @@ func parseVersion(line string) (version, error) {
|
||||
builder: m[6],
|
||||
}
|
||||
|
||||
// Split the version tag into tag and commit. This is old style
|
||||
// v1.2.3-something.4+11-g12345678 or newer with just dots
|
||||
// v1.2.3-something.4.11.g12345678 or v1.2.3-dev.11.g12345678.
|
||||
parts := []string{v.version}
|
||||
if strings.Contains(v.version, "+") {
|
||||
parts = strings.Split(v.version, "+")
|
||||
} else {
|
||||
idxs := gitExtraRE.FindStringIndex(v.version)
|
||||
if len(idxs) > 0 {
|
||||
parts = []string{v.version[:idxs[0]], v.version[idxs[0]+1:]}
|
||||
}
|
||||
}
|
||||
parts := strings.Split(v.version, "+")
|
||||
v.tag = parts[0]
|
||||
if len(parts) > 1 {
|
||||
fields := gitExtraSepRE.Split(parts[1], -1)
|
||||
fields := strings.Split(parts[1], "-")
|
||||
if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
|
||||
v.commit = fields[1][1:]
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -44,20 +44,6 @@ func TestParseVersion(t *testing.T) {
|
||||
extra: []string{"foo", "bar"},
|
||||
},
|
||||
},
|
||||
{
|
||||
longVersion: `syncthing v1.23.7-dev.26.gdf7b56ae-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]`,
|
||||
parsed: version{
|
||||
version: "v1.23.7-dev.26.gdf7b56ae-stversionextra",
|
||||
tag: "v1.23.7-dev",
|
||||
commit: "df7b56ae",
|
||||
codename: "Fermium Flea",
|
||||
runtime: "go1.20.5",
|
||||
goos: "darwin",
|
||||
goarch: "arm64",
|
||||
builder: "jb@ok.kastelo.net",
|
||||
extra: []string{"Some Wrapper", "purego", "stnoupgrade"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
@@ -73,7 +59,7 @@ func TestParseVersion(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseReport(t *testing.T) {
|
||||
bs, err := os.ReadFile("_testdata/panic.log")
|
||||
bs, err := ioutil.ReadFile("_testdata/panic.log")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -80,7 +80,7 @@ func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]b
|
||||
fmt.Println("Loading source:", resp.Status)
|
||||
return nil, 0
|
||||
}
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
data, err := ioutil.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
fmt.Println("Loading source:", err.Error())
|
||||
|
||||
@@ -7,16 +7,21 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type crashReceiver struct {
|
||||
store *diskStore
|
||||
sentry *sentryService
|
||||
dir string
|
||||
dsn string
|
||||
}
|
||||
|
||||
func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
@@ -39,64 +44,99 @@ func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// The location of the report on disk, compressed
|
||||
fullPath := filepath.Join(r.dir, r.dirFor(reportID), reportID) + ".gz"
|
||||
|
||||
switch req.Method {
|
||||
case http.MethodGet:
|
||||
r.serveGet(reportID, w, req)
|
||||
r.serveGet(fullPath, w, req)
|
||||
case http.MethodHead:
|
||||
r.serveHead(reportID, w, req)
|
||||
r.serveHead(fullPath, w, req)
|
||||
case http.MethodPut:
|
||||
r.servePut(reportID, w, req)
|
||||
r.servePut(reportID, fullPath, w, req)
|
||||
default:
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
// serveGet responds to GET requests by serving the uncompressed report.
|
||||
func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http.Request) {
|
||||
bs, err := r.store.Get(reportID)
|
||||
func (r *crashReceiver) serveGet(fullPath string, w http.ResponseWriter, _ *http.Request) {
|
||||
fd, err := os.Open(fullPath)
|
||||
if err != nil {
|
||||
http.Error(w, "Not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Write(bs)
|
||||
|
||||
defer fd.Close()
|
||||
gr, err := gzip.NewReader(fd)
|
||||
if err != nil {
|
||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
_, _ = io.Copy(w, gr) // best effort
|
||||
}
|
||||
|
||||
// serveHead responds to HEAD requests by checking if the named report
|
||||
// already exists in the system.
|
||||
func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) {
|
||||
if !r.store.Exists(reportID) {
|
||||
func (r *crashReceiver) serveHead(fullPath string, w http.ResponseWriter, _ *http.Request) {
|
||||
if _, err := os.Lstat(fullPath); err != nil {
|
||||
http.Error(w, "Not found", http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// servePut accepts and stores the given report.
|
||||
func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *http.Request) {
|
||||
result := "receive_failure"
|
||||
defer func() {
|
||||
metricCrashReportsTotal.WithLabelValues(result).Inc()
|
||||
}()
|
||||
func (r *crashReceiver) servePut(reportID, fullPath string, w http.ResponseWriter, req *http.Request) {
|
||||
// Ensure the destination directory exists
|
||||
if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
|
||||
log.Println("Creating directory:", err)
|
||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Read at most maxRequestSize of report data.
|
||||
log.Println("Receiving report", reportID)
|
||||
lr := io.LimitReader(req.Body, maxRequestSize)
|
||||
bs, err := io.ReadAll(lr)
|
||||
bs, err := ioutil.ReadAll(lr)
|
||||
if err != nil {
|
||||
log.Println("Reading report:", err)
|
||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
result = "success"
|
||||
// Compress the report for storage
|
||||
buf := new(bytes.Buffer)
|
||||
gw := gzip.NewWriter(buf)
|
||||
_, _ = gw.Write(bs) // can't fail
|
||||
gw.Close()
|
||||
|
||||
// Store the report
|
||||
if !r.store.Put(reportID, bs) {
|
||||
log.Println("Failed to store report (queue full):", reportID)
|
||||
result = "queue_failure"
|
||||
// Create an output file with the compressed report
|
||||
err = ioutil.WriteFile(fullPath, buf.Bytes(), 0644)
|
||||
if err != nil {
|
||||
log.Println("Saving report:", err)
|
||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Send the report to Sentry
|
||||
if !r.sentry.Send(reportID, userIDFor(req), bs) {
|
||||
log.Println("Failed to send report to sentry (queue full):", reportID)
|
||||
result = "sentry_failure"
|
||||
if r.dsn != "" {
|
||||
// Remote ID
|
||||
user := userIDFor(req)
|
||||
|
||||
go func() {
|
||||
// There's no need for the client to have to wait for this part.
|
||||
pkt, err := parseCrashReport(reportID, bs)
|
||||
if err != nil {
|
||||
log.Println("Failed to parse crash report:", err)
|
||||
return
|
||||
}
|
||||
if err := sendReport(r.dsn, pkt, user); err != nil {
|
||||
log.Println("Failed to send crash report:", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// 01234567890abcdef... => 01/23
|
||||
func (r *crashReceiver) dirFor(base string) string {
|
||||
return filepath.Join(base[0:2], base[2:4])
|
||||
}
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
// Copyright (C) 2021 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/sha256"
|
||||
)
|
||||
|
||||
// userIDFor returns a string we can use as the user ID for the purpose of
|
||||
// counting affected users. It's the truncated hash of a salt, the user
|
||||
// remote IP, and the current month.
|
||||
func userIDFor(req *http.Request) string {
|
||||
addr := req.RemoteAddr
|
||||
if fwd := req.Header.Get("x-forwarded-for"); fwd != "" {
|
||||
addr = fwd
|
||||
}
|
||||
if host, _, err := net.SplitHostPort(addr); err == nil {
|
||||
addr = host
|
||||
}
|
||||
now := time.Now().Format("200601")
|
||||
salt := "stcrashreporter"
|
||||
hash := sha256.Sum256([]byte(salt + addr + now))
|
||||
return fmt.Sprintf("%x", hash[:8])
|
||||
}
|
||||
|
||||
// 01234567890abcdef... => 01/23
|
||||
func dirFor(base string) string {
|
||||
return filepath.Join(base[0:2], base[2:4])
|
||||
}
|
||||
|
||||
func fullPathCompressed(root, reportID string) string {
|
||||
return filepath.Join(root, dirFor(reportID), reportID) + ".gz"
|
||||
}
|
||||
|
||||
func compressAndWrite(bs []byte, fullPath string) error {
|
||||
// Compress the report for storage
|
||||
buf := new(bytes.Buffer)
|
||||
gw := gzip.NewWriter(buf)
|
||||
_, _ = gw.Write(bs) // can't fail
|
||||
gw.Close()
|
||||
|
||||
// Create an output file with the compressed report
|
||||
return os.WriteFile(fullPath, buf.Bytes(), 0644)
|
||||
}
|
||||
@@ -8,15 +8,11 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
io "io"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net"
|
||||
@@ -29,7 +25,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/stringutil"
|
||||
)
|
||||
|
||||
// announcement is the format received from and sent to clients
|
||||
@@ -71,7 +66,7 @@ func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *apiSrv) Serve(_ context.Context) error {
|
||||
func (s *apiSrv) Serve(ctx context.Context) error {
|
||||
if s.useHTTP {
|
||||
listener, err := net.Listen("tcp", s.addr)
|
||||
if err != nil {
|
||||
@@ -81,10 +76,18 @@ func (s *apiSrv) Serve(_ context.Context) error {
|
||||
s.listener = listener
|
||||
} else {
|
||||
tlsCfg := &tls.Config{
|
||||
Certificates: []tls.Certificate{s.cert},
|
||||
ClientAuth: tls.RequestClientCert,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
NextProtos: []string{"h2", "http/1.1"},
|
||||
Certificates: []tls.Certificate{s.cert},
|
||||
ClientAuth: tls.RequestClientCert,
|
||||
SessionTicketsDisabled: true,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
CipherSuites: []uint16{
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
|
||||
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
|
||||
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
|
||||
},
|
||||
}
|
||||
|
||||
tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
|
||||
@@ -102,7 +105,6 @@ func (s *apiSrv) Serve(_ context.Context) error {
|
||||
ReadTimeout: httpReadTimeout,
|
||||
WriteTimeout: httpWriteTimeout,
|
||||
MaxHeaderBytes: httpMaxHeaderBytes,
|
||||
ErrorLog: log.New(io.Discard, "", 0),
|
||||
}
|
||||
|
||||
err := srv.Serve(s.listener)
|
||||
@@ -112,6 +114,8 @@ func (s *apiSrv) Serve(_ context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
var topCtx = context.Background()
|
||||
|
||||
func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
|
||||
t0 := time.Now()
|
||||
|
||||
@@ -124,10 +128,10 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
|
||||
}()
|
||||
|
||||
reqID := requestID(rand.Int63())
|
||||
req = req.WithContext(context.WithValue(req.Context(), idKey, reqID))
|
||||
ctx := context.WithValue(topCtx, idKey, reqID)
|
||||
|
||||
if debug {
|
||||
log.Println(reqID, req.Method, req.URL, req.Proto)
|
||||
log.Println(reqID, req.Method, req.URL)
|
||||
}
|
||||
|
||||
remoteAddr := &net.TCPAddr{
|
||||
@@ -136,12 +140,7 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
if s.useHTTP {
|
||||
// X-Forwarded-For can have multiple client IPs; split using the comma separator
|
||||
forwardIP, _, _ := strings.Cut(req.Header.Get("X-Forwarded-For"), ",")
|
||||
|
||||
// net.ParseIP will return nil if leading/trailing whitespace exists; use strings.TrimSpace()
|
||||
remoteAddr.IP = net.ParseIP(strings.TrimSpace(forwardIP))
|
||||
|
||||
remoteAddr.IP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
|
||||
if parsedPort, err := strconv.ParseInt(req.Header.Get("X-Client-Port"), 10, 0); err == nil {
|
||||
remoteAddr.Port = int(parsedPort)
|
||||
}
|
||||
@@ -158,17 +157,17 @@ func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
switch req.Method {
|
||||
case http.MethodGet:
|
||||
s.handleGET(lw, req)
|
||||
case http.MethodPost:
|
||||
s.handlePOST(remoteAddr, lw, req)
|
||||
case "GET":
|
||||
s.handleGET(ctx, lw, req)
|
||||
case "POST":
|
||||
s.handlePOST(ctx, remoteAddr, lw, req)
|
||||
default:
|
||||
http.Error(lw, "Method Not Allowed", http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
|
||||
reqID := req.Context().Value(idKey).(requestID)
|
||||
func (s *apiSrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
|
||||
reqID := ctx.Value(idKey).(requestID)
|
||||
|
||||
deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
|
||||
if err != nil {
|
||||
@@ -212,39 +211,28 @@ func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
|
||||
s.db.put(key, rec)
|
||||
}
|
||||
|
||||
afterS := notFoundRetryAfterSeconds(int(misses))
|
||||
retryAfterHistogram.Observe(float64(afterS))
|
||||
w.Header().Set("Retry-After", strconv.Itoa(afterS))
|
||||
w.Header().Set("Retry-After", notFoundRetryAfterString(int(misses)))
|
||||
http.Error(w, "Not Found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
lookupRequestsTotal.WithLabelValues("success").Inc()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
var bw io.Writer = w
|
||||
|
||||
// Use compression if the client asks for it
|
||||
if strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") {
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
gw := gzip.NewWriter(bw)
|
||||
defer gw.Close()
|
||||
bw = gw
|
||||
}
|
||||
|
||||
json.NewEncoder(bw).Encode(announcement{
|
||||
Seen: time.Unix(0, rec.Seen).Truncate(time.Second),
|
||||
bs, _ := json.Marshal(announcement{
|
||||
Seen: time.Unix(0, rec.Seen),
|
||||
Addresses: addressStrs(rec.Addresses),
|
||||
})
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write(bs)
|
||||
}
|
||||
|
||||
func (s *apiSrv) handlePOST(remoteAddr *net.TCPAddr, w http.ResponseWriter, req *http.Request) {
|
||||
reqID := req.Context().Value(idKey).(requestID)
|
||||
func (s *apiSrv) handlePOST(ctx context.Context, remoteAddr *net.TCPAddr, w http.ResponseWriter, req *http.Request) {
|
||||
reqID := ctx.Value(idKey).(requestID)
|
||||
|
||||
rawCert, err := certificateBytes(req)
|
||||
if err != nil {
|
||||
rawCert := certificateBytes(req)
|
||||
if rawCert == nil {
|
||||
if debug {
|
||||
log.Println(reqID, "no certificates:", err)
|
||||
log.Println(reqID, "no certificates")
|
||||
}
|
||||
announceRequestsTotal.WithLabelValues("no_certificate").Inc()
|
||||
w.Header().Set("Retry-After", errorRetryAfterString())
|
||||
@@ -312,13 +300,13 @@ func (s *apiSrv) handleAnnounce(deviceID protocol.DeviceID, addresses []string)
|
||||
return s.db.merge(key, dbAddrs, seen)
|
||||
}
|
||||
|
||||
func handlePing(w http.ResponseWriter, _ *http.Request) {
|
||||
func handlePing(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
|
||||
func certificateBytes(req *http.Request) ([]byte, error) {
|
||||
func certificateBytes(req *http.Request) []byte {
|
||||
if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
|
||||
return req.TLS.PeerCertificates[0].Raw, nil
|
||||
return req.TLS.PeerCertificates[0].Raw
|
||||
}
|
||||
|
||||
var bs []byte
|
||||
@@ -331,7 +319,7 @@ func certificateBytes(req *http.Request) ([]byte, error) {
|
||||
hdr, err := url.QueryUnescape(hdr)
|
||||
if err != nil {
|
||||
// Decoding failed
|
||||
return nil, err
|
||||
return nil
|
||||
}
|
||||
|
||||
bs = []byte(hdr)
|
||||
@@ -350,27 +338,15 @@ func certificateBytes(req *http.Request) ([]byte, error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if hdr := req.Header.Get("X-Tls-Client-Cert-Der-Base64"); hdr != "" {
|
||||
// Caddy {tls_client_certificate_der_base64}
|
||||
hdr, err := base64.StdEncoding.DecodeString(hdr)
|
||||
if err != nil {
|
||||
// Decoding failed
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bs = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: hdr})
|
||||
} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
|
||||
// Traefik 2 passtlsclientcert
|
||||
//
|
||||
// The certificate is in PEM format, maybe with URL encoding
|
||||
// (depends on Traefik version) but without newlines and start/end
|
||||
// statements. We need to decode, reinstate the newlines every 64
|
||||
// The certificate is in PEM format with url encoding but without newlines
|
||||
// and start/end statements. We need to decode, reinstate the newlines every 64
|
||||
// character and add statements for the PEM decoder
|
||||
|
||||
if strings.Contains(hdr, "%") {
|
||||
if unesc, err := url.QueryUnescape(hdr); err == nil {
|
||||
hdr = unesc
|
||||
}
|
||||
hdr, err := url.QueryUnescape(hdr)
|
||||
if err != nil {
|
||||
// Decoding failed
|
||||
return nil
|
||||
}
|
||||
|
||||
for i := 64; i < len(hdr); i += 65 {
|
||||
@@ -378,21 +354,21 @@ func certificateBytes(req *http.Request) ([]byte, error) {
|
||||
}
|
||||
|
||||
hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
|
||||
hdr += "\n-----END CERTIFICATE-----\n"
|
||||
hdr = hdr + "\n-----END CERTIFICATE-----\n"
|
||||
bs = []byte(hdr)
|
||||
}
|
||||
|
||||
if bs == nil {
|
||||
return nil, errors.New("empty certificate header")
|
||||
return nil
|
||||
}
|
||||
|
||||
block, _ := pem.Decode(bs)
|
||||
if block == nil {
|
||||
// Decoding failed
|
||||
return nil, errors.New("certificate decode result is empty")
|
||||
return nil
|
||||
}
|
||||
|
||||
return block.Bytes, nil
|
||||
return block.Bytes
|
||||
}
|
||||
|
||||
// fixupAddresses checks the list of addresses, removing invalid ones and
|
||||
@@ -417,13 +393,13 @@ func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
|
||||
continue
|
||||
}
|
||||
|
||||
if host == "" || ip.IsUnspecified() {
|
||||
if remote != nil {
|
||||
if remote != nil {
|
||||
if host == "" || ip.IsUnspecified() {
|
||||
// Replace the unspecified IP with the request source.
|
||||
|
||||
// ... unless the request source is the loopback address or
|
||||
// multicast/unspecified (can't happen, really).
|
||||
if remote.IP == nil || remote.IP.IsLoopback() || remote.IP.IsMulticast() || remote.IP.IsUnspecified() {
|
||||
if remote.IP.IsLoopback() || remote.IP.IsMulticast() || remote.IP.IsUnspecified() {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -439,22 +415,11 @@ func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
|
||||
}
|
||||
|
||||
host = remote.IP.String()
|
||||
|
||||
} else {
|
||||
// remote is nil, unable to determine host IP
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// If zero port was specified, use remote port.
|
||||
if port == "0" {
|
||||
if remote != nil && remote.Port > 0 {
|
||||
// use remote port
|
||||
port = strconv.Itoa(remote.Port)
|
||||
} else {
|
||||
// unable to determine remote port
|
||||
continue
|
||||
// If zero port was specified, use remote port.
|
||||
if port == "0" && remote.Port > 0 {
|
||||
port = fmt.Sprintf("%d", remote.Port)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -462,9 +427,6 @@ func fixupAddresses(remote *net.TCPAddr, addresses []string) []string {
|
||||
fixed = append(fixed, uri.String())
|
||||
}
|
||||
|
||||
// Remove duplicate addresses
|
||||
fixed = stringutil.UniqueTrimmedStrings(fixed)
|
||||
|
||||
return fixed
|
||||
}
|
||||
|
||||
@@ -494,13 +456,13 @@ func errorRetryAfterString() string {
|
||||
return strconv.Itoa(errorRetryAfterSeconds + rand.Intn(errorRetryFuzzSeconds))
|
||||
}
|
||||
|
||||
func notFoundRetryAfterSeconds(misses int) int {
|
||||
func notFoundRetryAfterString(misses int) string {
|
||||
retryAfterS := notFoundRetryMinSeconds + notFoundRetryIncSeconds*misses
|
||||
if retryAfterS > notFoundRetryMaxSeconds {
|
||||
retryAfterS = notFoundRetryMaxSeconds
|
||||
}
|
||||
retryAfterS += rand.Intn(notFoundRetryFuzzSeconds)
|
||||
return retryAfterS
|
||||
return strconv.Itoa(retryAfterS)
|
||||
}
|
||||
|
||||
func reannounceAfterString() string {
|
||||
|
||||
@@ -69,14 +69,6 @@ func TestFixupAddresses(t *testing.T) {
|
||||
remote: addr("123.123.123.123", 9000),
|
||||
in: []string{"tcp://44.44.44.44:0"},
|
||||
out: []string{"tcp://44.44.44.44:9000"},
|
||||
}, { // remote ip nil
|
||||
remote: addr("", 9000),
|
||||
in: []string{"tcp://:22000", "tcp://44.44.44.44:9000"},
|
||||
out: []string{"tcp://44.44.44.44:9000"},
|
||||
}, { // remote port 0
|
||||
remote: addr("123.123.123.123", 0),
|
||||
in: []string{"tcp://:22000", "tcp://44.44.44.44"},
|
||||
out: []string{"tcp://123.123.123.123:22000"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -12,14 +12,10 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net"
|
||||
"net/url"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/sliceutil"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/syndtr/goleveldb/leveldb/storage"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
@@ -58,18 +54,6 @@ func newLevelDBStore(dir string) (*levelDBStore, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newMemoryLevelDBStore() (*levelDBStore, error) {
|
||||
db, err := leveldb.Open(storage.NewMemStorage(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &levelDBStore{
|
||||
db: db,
|
||||
inbox: make(chan func(), 16),
|
||||
clock: defaultClock{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *levelDBStore) put(key string, rec DatabaseRecord) error {
|
||||
t0 := time.Now()
|
||||
defer func() {
|
||||
@@ -220,7 +204,7 @@ func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- stru
|
||||
cutoff24h := t0.Add(-24 * time.Hour).UnixNano()
|
||||
cutoff1w := t0.Add(-7 * 24 * time.Hour).UnixNano()
|
||||
cutoff2Mon := t0.Add(-60 * 24 * time.Hour).UnixNano()
|
||||
current, currentIPv4, currentIPv6, last24h, last1w, inactive, errors := 0, 0, 0, 0, 0, 0, 0
|
||||
current, last24h, last1w, inactive, errors := 0, 0, 0, 0, 0
|
||||
|
||||
iter := s.db.NewIterator(&util.Range{}, nil)
|
||||
for iter.Next() {
|
||||
@@ -235,35 +219,9 @@ func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- stru
|
||||
// If there are addresses that have not expired it's a current
|
||||
// record, otherwise account it based on when it was last seen
|
||||
// (last 24 hours or last week) or finally as inactice.
|
||||
addrs := expire(rec.Addresses, nowNanos)
|
||||
switch {
|
||||
case len(addrs) > 0:
|
||||
case len(expire(rec.Addresses, nowNanos)) > 0:
|
||||
current++
|
||||
seenIPv4, seenIPv6 := false, false
|
||||
for _, addr := range addrs {
|
||||
uri, err := url.Parse(addr.Address)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
host, _, err := net.SplitHostPort(uri.Host)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if ip := net.ParseIP(host); ip != nil && ip.To4() != nil {
|
||||
seenIPv4 = true
|
||||
} else if ip != nil {
|
||||
seenIPv6 = true
|
||||
}
|
||||
if seenIPv4 && seenIPv6 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if seenIPv4 {
|
||||
currentIPv4++
|
||||
}
|
||||
if seenIPv6 {
|
||||
currentIPv6++
|
||||
}
|
||||
case rec.Seen > cutoff24h:
|
||||
last24h++
|
||||
case rec.Seen > cutoff1w:
|
||||
@@ -287,8 +245,6 @@ func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- stru
|
||||
iter.Release()
|
||||
|
||||
databaseKeys.WithLabelValues("current").Set(float64(current))
|
||||
databaseKeys.WithLabelValues("currentIPv4").Set(float64(currentIPv4))
|
||||
databaseKeys.WithLabelValues("currentIPv6").Set(float64(currentIPv6))
|
||||
databaseKeys.WithLabelValues("last24h").Set(float64(last24h))
|
||||
databaseKeys.WithLabelValues("last1w").Set(float64(last1w))
|
||||
databaseKeys.WithLabelValues("inactive").Set(float64(inactive))
|
||||
@@ -383,7 +339,14 @@ func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
|
||||
i := 0
|
||||
for i < len(addrs) {
|
||||
if addrs[i].Expires < now {
|
||||
addrs = sliceutil.RemoveAndZero(addrs, i)
|
||||
// This item is expired. Replace it with the last in the list
|
||||
// (noop if we are at the last item).
|
||||
addrs[i] = addrs[len(addrs)-1]
|
||||
// Wipe the last item of the list to release references to
|
||||
// strings and stuff.
|
||||
addrs[len(addrs)-1] = DatabaseAddress{}
|
||||
// Shorten the slice.
|
||||
addrs = addrs[:len(addrs)-1]
|
||||
continue
|
||||
}
|
||||
i++
|
||||
|
||||
@@ -9,12 +9,15 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDatabaseGetSet(t *testing.T) {
|
||||
db, err := newMemoryLevelDBStore()
|
||||
os.RemoveAll("_database")
|
||||
defer os.RemoveAll("_database")
|
||||
db, err := newLevelDBStore("_database")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -116,7 +119,7 @@ func TestDatabaseGetSet(t *testing.T) {
|
||||
|
||||
// Put a record with misses
|
||||
|
||||
rec = DatabaseRecord{Misses: 42, Missed: tc.Now().UnixNano()}
|
||||
rec = DatabaseRecord{Misses: 42}
|
||||
if err := db.put("efgh", rec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -185,7 +188,7 @@ func TestFilter(t *testing.T) {
|
||||
},
|
||||
{
|
||||
a: []DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
|
||||
b: []DatabaseAddress{{Address: "b", Expires: 15}, {Address: "d", Expires: 15}},
|
||||
b: []DatabaseAddress{{Address: "d", Expires: 15}, {Address: "b", Expires: 15}}, // gets reordered
|
||||
},
|
||||
}
|
||||
|
||||
@@ -206,6 +209,5 @@ func (t *testClock) wind(d time.Duration) {
|
||||
}
|
||||
|
||||
func (t *testClock) Now() time.Time {
|
||||
t.now = t.now.Add(time.Nanosecond)
|
||||
return t.now
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -65,7 +64,9 @@ var levelDBOptions = &opt.Options{
|
||||
WriteBuffer: 32 << 20, // default 4<<20
|
||||
}
|
||||
|
||||
var debug = false
|
||||
var (
|
||||
debug = false
|
||||
)
|
||||
|
||||
func main() {
|
||||
var listen string
|
||||
@@ -75,26 +76,20 @@ func main() {
|
||||
var replicationPeers string
|
||||
var certFile string
|
||||
var keyFile string
|
||||
var replCertFile string
|
||||
var replKeyFile string
|
||||
var useHTTP bool
|
||||
var largeDB bool
|
||||
|
||||
log.SetOutput(os.Stdout)
|
||||
log.SetFlags(0)
|
||||
|
||||
flag.StringVar(&certFile, "cert", "./cert.pem", "Certificate file")
|
||||
flag.StringVar(&keyFile, "key", "./key.pem", "Key file")
|
||||
flag.StringVar(&dir, "db-dir", "./discovery.db", "Database directory")
|
||||
flag.BoolVar(&debug, "debug", false, "Print debug output")
|
||||
flag.BoolVar(&useHTTP, "http", false, "Listen on HTTP (behind an HTTPS proxy)")
|
||||
flag.StringVar(&listen, "listen", ":8443", "Listen address")
|
||||
flag.StringVar(&keyFile, "key", "./key.pem", "Key file")
|
||||
flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
|
||||
flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
|
||||
flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
|
||||
flag.StringVar(&replCertFile, "replication-cert", "", "Certificate file for replication")
|
||||
flag.StringVar(&replKeyFile, "replication-key", "", "Key file for replication")
|
||||
flag.BoolVar(&largeDB, "large-db", false, "Use larger database settings")
|
||||
showVersion := flag.Bool("version", false, "Show version")
|
||||
flag.Parse()
|
||||
|
||||
@@ -103,17 +98,6 @@ func main() {
|
||||
return
|
||||
}
|
||||
|
||||
buildInfo.WithLabelValues(build.Version, runtime.Version(), build.User, build.Date.UTC().Format("2006-01-02T15:04:05Z")).Set(1)
|
||||
|
||||
if largeDB {
|
||||
levelDBOptions.BlockCacheCapacity = 64 << 20
|
||||
levelDBOptions.BlockSize = 64 << 10
|
||||
levelDBOptions.CompactionTableSize = 16 << 20
|
||||
levelDBOptions.CompactionTableSizeMultiplier = 2.0
|
||||
levelDBOptions.WriteBuffer = 64 << 20
|
||||
levelDBOptions.CompactionL0Trigger = 8
|
||||
}
|
||||
|
||||
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
|
||||
if os.IsNotExist(err) {
|
||||
log.Println("Failed to load keypair. Generating one, this might take a while...")
|
||||
@@ -127,26 +111,13 @@ func main() {
|
||||
devID := protocol.NewDeviceID(cert.Certificate[0])
|
||||
log.Println("Server device ID is", devID)
|
||||
|
||||
replCert := cert
|
||||
if replCertFile != "" && replKeyFile != "" {
|
||||
replCert, err = tls.LoadX509KeyPair(replCertFile, replKeyFile)
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to load replication keypair:", err)
|
||||
}
|
||||
}
|
||||
replDevID := protocol.NewDeviceID(replCert.Certificate[0])
|
||||
log.Println("Replication device ID is", replDevID)
|
||||
|
||||
// Parse the replication specs, if any.
|
||||
var allowedReplicationPeers []protocol.DeviceID
|
||||
var replicationDestinations []string
|
||||
parts := strings.Split(replicationPeers, ",")
|
||||
for _, part := range parts {
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
fields := strings.Split(part, "@")
|
||||
|
||||
switch len(fields) {
|
||||
case 2:
|
||||
// This is an id@address specification. Grab the address for the
|
||||
@@ -166,9 +137,6 @@ func main() {
|
||||
if err != nil {
|
||||
log.Fatalln("Parsing device ID:", err)
|
||||
}
|
||||
if id == protocol.EmptyDeviceID {
|
||||
log.Fatalf("Missing device ID for peer in %q", part)
|
||||
}
|
||||
allowedReplicationPeers = append(allowedReplicationPeers, id)
|
||||
|
||||
default:
|
||||
@@ -191,14 +159,14 @@ func main() {
|
||||
// Start any replication senders.
|
||||
var repl replicationMultiplexer
|
||||
for _, dst := range replicationDestinations {
|
||||
rs := newReplicationSender(dst, replCert, allowedReplicationPeers)
|
||||
rs := newReplicationSender(dst, cert, allowedReplicationPeers)
|
||||
main.Add(rs)
|
||||
repl = append(repl, rs)
|
||||
}
|
||||
|
||||
// If we have replication configured, start the replication listener.
|
||||
if len(allowedReplicationPeers) > 0 {
|
||||
rl := newReplicationListener(replicationListen, replCert, allowedReplicationPeers, db)
|
||||
rl := newReplicationListener(replicationListen, cert, allowedReplicationPeers, db)
|
||||
main.Add(rl)
|
||||
}
|
||||
|
||||
|
||||
@@ -19,11 +19,8 @@ import (
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
)
|
||||
|
||||
const (
|
||||
replicationReadTimeout = time.Minute
|
||||
replicationWriteTimeout = 30 * time.Second
|
||||
replicationHeartbeatInterval = time.Second * 30
|
||||
)
|
||||
const replicationReadTimeout = time.Minute
|
||||
const replicationHeartbeatInterval = time.Second * 30
|
||||
|
||||
type replicator interface {
|
||||
send(key string, addrs []DatabaseAddress, seen int64)
|
||||
@@ -71,12 +68,6 @@ func (s *replicationSender) Serve(ctx context.Context) error {
|
||||
conn.Close()
|
||||
}()
|
||||
|
||||
// The replication stream is not especially latency sensitive, but it is
|
||||
// quite a lot of data in small writes. Make it more efficient.
|
||||
if tcpc, ok := conn.NetConn().(*net.TCPConn); ok {
|
||||
_ = tcpc.SetNoDelay(false)
|
||||
}
|
||||
|
||||
// Get the other side device ID.
|
||||
remoteID, err := deviceID(conn)
|
||||
if err != nil {
|
||||
@@ -125,11 +116,11 @@ func (s *replicationSender) Serve(ctx context.Context) error {
|
||||
binary.BigEndian.PutUint32(buf, uint32(n))
|
||||
|
||||
// Send
|
||||
conn.SetWriteDeadline(time.Now().Add(replicationWriteTimeout))
|
||||
conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
|
||||
if _, err := conn.Write(buf[:4+n]); err != nil {
|
||||
replicationSendsTotal.WithLabelValues("error").Inc()
|
||||
log.Println("Replication write:", err)
|
||||
// Yes, we are losing the replication event here.
|
||||
// Yes, we are loosing the replication event here.
|
||||
return err
|
||||
}
|
||||
replicationSendsTotal.WithLabelValues("success").Inc()
|
||||
@@ -144,7 +135,7 @@ func (s *replicationSender) String() string {
|
||||
return fmt.Sprintf("replicationSender(%q)", s.dst)
|
||||
}
|
||||
|
||||
func (s *replicationSender) send(key string, ps []DatabaseAddress, _ int64) {
|
||||
func (s *replicationSender) send(key string, ps []DatabaseAddress, seen int64) {
|
||||
item := ReplicationRecord{
|
||||
Key: key,
|
||||
Addresses: ps,
|
||||
|
||||
@@ -10,18 +10,9 @@ import (
|
||||
"os"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
)
|
||||
|
||||
var (
|
||||
buildInfo = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "discovery",
|
||||
Name: "build_info",
|
||||
Help: "A metric with a constant '1' value labeled by version, goversion, builduser and builddate from which stdiscosrv was built.",
|
||||
}, []string{"version", "goversion", "builduser", "builddate"})
|
||||
|
||||
apiRequestsTotal = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: "syncthing",
|
||||
@@ -98,14 +89,6 @@ var (
|
||||
Help: "Latency of database operations.",
|
||||
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
|
||||
}, []string{"operation"})
|
||||
|
||||
retryAfterHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "discovery",
|
||||
Name: "retry_after_seconds",
|
||||
Help: "Retry-After header value in seconds.",
|
||||
Buckets: prometheus.ExponentialBuckets(60, 2, 7), // 60, 120, 240, 480, 960, 1920, 3840
|
||||
})
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -120,15 +103,13 @@ const (
|
||||
)
|
||||
|
||||
func init() {
|
||||
prometheus.MustRegister(buildInfo,
|
||||
apiRequestsTotal, apiRequestsSeconds,
|
||||
prometheus.MustRegister(apiRequestsTotal, apiRequestsSeconds,
|
||||
lookupRequestsTotal, announceRequestsTotal,
|
||||
replicationSendsTotal, replicationRecvsTotal,
|
||||
databaseKeys, databaseStatisticsSeconds,
|
||||
databaseOperations, databaseOperationSeconds,
|
||||
retryAfterHistogram)
|
||||
databaseOperations, databaseOperationSeconds)
|
||||
|
||||
processCollectorOpts := collectors.ProcessCollectorOpts{
|
||||
processCollectorOpts := prometheus.ProcessCollectorOpts{
|
||||
Namespace: "syncthing_discovery",
|
||||
PidFn: func() (int, error) {
|
||||
return os.Getpid(), nil
|
||||
@@ -136,6 +117,7 @@ func init() {
|
||||
}
|
||||
|
||||
prometheus.MustRegister(
|
||||
collectors.NewProcessCollector(processCollectorOpts),
|
||||
prometheus.NewProcessCollector(processCollectorOpts),
|
||||
)
|
||||
|
||||
}
|
||||
|
||||
@@ -84,7 +84,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
|
||||
}
|
||||
|
||||
func checkServer(deviceID protocol.DeviceID, server string) checkResult {
|
||||
disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger, nil)
|
||||
disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger)
|
||||
if err != nil {
|
||||
return checkResult{error: err}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
// Command stfindignored lists ignored files under a given folder root.
|
||||
// Commmand stfindignored lists ignored files under a given folder root.
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -66,7 +66,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
|
||||
}
|
||||
|
||||
func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
|
||||
src := io.LimitReader(&infiniteReader{fd}, s)
|
||||
src := io.LimitReader(&inifiteReader{fd}, s)
|
||||
dst, err := os.Create(p1)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -105,11 +105,11 @@ func readRand(bs []byte) (int, error) {
|
||||
return len(bs), nil
|
||||
}
|
||||
|
||||
type infiniteReader struct {
|
||||
type inifiteReader struct {
|
||||
rd io.ReadSeeker
|
||||
}
|
||||
|
||||
func (i *infiniteReader) Read(bs []byte) (int, error) {
|
||||
func (i *inifiteReader) Read(bs []byte) (int, error) {
|
||||
n, err := i.rd.Read(bs)
|
||||
if err == io.EOF {
|
||||
err = nil
|
||||
|
||||
@@ -4,8 +4,7 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
//go:build noassets
|
||||
// +build noassets
|
||||
//+build noassets
|
||||
|
||||
package auto
|
||||
|
||||
|
||||
@@ -237,7 +237,7 @@
|
||||
uptimeSeconds: 0,
|
||||
};
|
||||
$scope.map = L.map('map').setView([40.90296, 1.90925], 2);
|
||||
L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png',
|
||||
L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',
|
||||
{
|
||||
attribution: 'Leaflet',
|
||||
maxZoom: 17
|
||||
|
||||
@@ -3,12 +3,15 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -17,13 +20,11 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
"github.com/syncthing/syncthing/lib/httpcache"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
|
||||
"github.com/golang/groupcache/lru"
|
||||
"github.com/oschwald/geoip2-golang"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
@@ -33,6 +34,7 @@ import (
|
||||
"github.com/syncthing/syncthing/lib/relay/client"
|
||||
"github.com/syncthing/syncthing/lib/sync"
|
||||
"github.com/syncthing/syncthing/lib/tlsutil"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
type location struct {
|
||||
@@ -98,13 +100,27 @@ var (
|
||||
dir string
|
||||
evictionTime = time.Hour
|
||||
debug bool
|
||||
getLRUSize = 10 << 10
|
||||
getLimitBurst = 10
|
||||
getLimitAvg = 2
|
||||
postLRUSize = 1 << 10
|
||||
postLimitBurst = 2
|
||||
postLimitAvg = 2
|
||||
getLimit time.Duration
|
||||
postLimit time.Duration
|
||||
permRelaysFile string
|
||||
ipHeader string
|
||||
geoipPath string
|
||||
proto string
|
||||
statsRefresh = time.Minute
|
||||
requestQueueLen = 64
|
||||
requestProcessors = 8
|
||||
statsRefresh = time.Minute / 2
|
||||
requestQueueLen = 10
|
||||
requestProcessors = 1
|
||||
|
||||
getMut = sync.NewMutex()
|
||||
getLRUCache *lru.Cache
|
||||
|
||||
postMut = sync.NewMutex()
|
||||
postLRUCache *lru.Cache
|
||||
|
||||
requests chan request
|
||||
|
||||
@@ -112,7 +128,6 @@ var (
|
||||
knownRelays = make([]*relay, 0)
|
||||
permanentRelays = make([]*relay, 0)
|
||||
evictionTimers = make(map[string]*time.Timer)
|
||||
globalBlocklist = newErrorTracker(1000)
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -127,8 +142,13 @@ func main() {
|
||||
flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
|
||||
flag.BoolVar(&debug, "debug", debug, "Enable debug output")
|
||||
flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
|
||||
flag.IntVar(&getLRUSize, "get-limit-cache", getLRUSize, "Get request limiter cache size")
|
||||
flag.IntVar(&getLimitAvg, "get-limit-avg", getLimitAvg, "Allowed average get request rate, per 10 s")
|
||||
flag.IntVar(&getLimitBurst, "get-limit-burst", getLimitBurst, "Allowed burst get requests")
|
||||
flag.IntVar(&postLRUSize, "post-limit-cache", postLRUSize, "Post request limiter cache size")
|
||||
flag.IntVar(&postLimitAvg, "post-limit-avg", postLimitAvg, "Allowed average post request rate, per minute")
|
||||
flag.IntVar(&postLimitBurst, "post-limit-burst", postLimitBurst, "Allowed burst post requests")
|
||||
flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
|
||||
flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays")
|
||||
flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
|
||||
flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
|
||||
flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
|
||||
@@ -140,6 +160,12 @@ func main() {
|
||||
|
||||
requests = make(chan request, requestQueueLen)
|
||||
|
||||
getLimit = 10 * time.Second / time.Duration(getLimitAvg)
|
||||
postLimit = time.Minute / time.Duration(postLimitAvg)
|
||||
|
||||
getLRUCache = lru.New(getLRUSize)
|
||||
postLRUCache = lru.New(postLRUSize)
|
||||
|
||||
var listener net.Listener
|
||||
var err error
|
||||
|
||||
@@ -215,7 +241,7 @@ func main() {
|
||||
|
||||
handler := http.NewServeMux()
|
||||
handler.HandleFunc("/", handleAssets)
|
||||
handler.Handle("/endpoint", httpcache.SinglePath(http.HandlerFunc(handleRequest), 15*time.Second))
|
||||
handler.HandleFunc("/endpoint", handleRequest)
|
||||
handler.HandleFunc("/metrics", handleMetrics)
|
||||
|
||||
srv := http.Server{
|
||||
@@ -266,17 +292,21 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
|
||||
}()
|
||||
|
||||
if ipHeader != "" {
|
||||
hdr := r.Header.Get(ipHeader)
|
||||
fields := strings.Split(hdr, ",")
|
||||
if len(fields) > 0 {
|
||||
r.RemoteAddr = strings.TrimSpace(fields[len(fields)-1])
|
||||
}
|
||||
r.RemoteAddr = r.Header.Get(ipHeader)
|
||||
}
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
switch r.Method {
|
||||
case "GET":
|
||||
if limit(r.RemoteAddr, getLRUCache, getMut, getLimit, getLimitBurst) {
|
||||
w.WriteHeader(httpStatusEnhanceYourCalm)
|
||||
return
|
||||
}
|
||||
handleGetRequest(w, r)
|
||||
case "POST":
|
||||
if limit(r.RemoteAddr, postLRUCache, postMut, postLimit, postLimitBurst) {
|
||||
w.WriteHeader(httpStatusEnhanceYourCalm)
|
||||
return
|
||||
}
|
||||
handlePostRequest(w, r)
|
||||
default:
|
||||
if debug {
|
||||
@@ -298,28 +328,20 @@ func handleGetRequest(rw http.ResponseWriter, r *http.Request) {
|
||||
// Shuffle
|
||||
rand.Shuffle(relays)
|
||||
|
||||
_ = json.NewEncoder(rw).Encode(map[string][]*relay{
|
||||
w := io.Writer(rw)
|
||||
if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
|
||||
rw.Header().Set("Content-Encoding", "gzip")
|
||||
gw := gzip.NewWriter(rw)
|
||||
defer gw.Close()
|
||||
w = gw
|
||||
}
|
||||
|
||||
_ = json.NewEncoder(w).Encode(map[string][]*relay{
|
||||
"relays": relays,
|
||||
})
|
||||
}
|
||||
|
||||
func handlePostRequest(w http.ResponseWriter, r *http.Request) {
|
||||
// Get the IP address of the client
|
||||
rhost := r.RemoteAddr
|
||||
if host, _, err := net.SplitHostPort(rhost); err == nil {
|
||||
rhost = host
|
||||
}
|
||||
|
||||
// Check the black list. A client is blacklisted if their last 10
|
||||
// attempts to join have all failed. The "Unauthorized" status return
|
||||
// causes strelaysrv to cease attempting to join.
|
||||
if globalBlocklist.IsBlocked(rhost) {
|
||||
log.Println("Rejected blocked client", rhost)
|
||||
http.Error(w, "Too many errors", http.StatusUnauthorized)
|
||||
globalBlocklist.ClearErrors(rhost)
|
||||
return
|
||||
}
|
||||
|
||||
var relayCert *x509.Certificate
|
||||
if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
|
||||
relayCert = r.TLS.PeerCertificates[0]
|
||||
@@ -347,11 +369,6 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Canonicalize the URL. In particular, parse and re-encode the query
|
||||
// string so that it's guaranteed to be valid.
|
||||
uri.RawQuery = uri.Query().Encode()
|
||||
newRelay.URL = uri.String()
|
||||
|
||||
if relayCert != nil {
|
||||
advertisedId := uri.Query().Get("id")
|
||||
idFromCert := protocol.NewDeviceID(relayCert.Raw).String()
|
||||
@@ -371,6 +388,12 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Get the IP address of the client
|
||||
rhost := r.RemoteAddr
|
||||
if host, _, err := net.SplitHostPort(rhost); err == nil {
|
||||
rhost = host
|
||||
}
|
||||
|
||||
ip := net.ParseIP(host)
|
||||
// The client did not provide an IP address, use the IP address of the client.
|
||||
if ip == nil || ip.IsUnspecified() {
|
||||
@@ -402,14 +425,10 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
|
||||
case requests <- request{&newRelay, reschan, prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("queue"))}:
|
||||
result := <-reschan
|
||||
if result.err != nil {
|
||||
log.Println("Join from", r.RemoteAddr, "failed:", result.err)
|
||||
globalBlocklist.AddError(rhost)
|
||||
relayTestsTotal.WithLabelValues("failed").Inc()
|
||||
http.Error(w, result.err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
log.Println("Join from", r.RemoteAddr, "succeeded")
|
||||
globalBlocklist.ClearErrors(rhost)
|
||||
relayTestsTotal.WithLabelValues("success").Inc()
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
json.NewEncoder(w).Encode(map[string]time.Duration{
|
||||
@@ -523,8 +542,25 @@ func evict(relay *relay) func() {
|
||||
}
|
||||
}
|
||||
|
||||
func limit(addr string, cache *lru.Cache, lock sync.Mutex, intv time.Duration, burst int) bool {
|
||||
if host, _, err := net.SplitHostPort(addr); err == nil {
|
||||
addr = host
|
||||
}
|
||||
|
||||
lock.Lock()
|
||||
v, _ := cache.Get(addr)
|
||||
bkt, ok := v.(*rate.Limiter)
|
||||
if !ok {
|
||||
bkt = rate.NewLimiter(rate.Every(intv), burst)
|
||||
cache.Add(addr, bkt)
|
||||
}
|
||||
lock.Unlock()
|
||||
|
||||
return !bkt.Allow()
|
||||
}
|
||||
|
||||
func loadRelays(file string) []*relay {
|
||||
content, err := os.ReadFile(file)
|
||||
content, err := ioutil.ReadFile(file)
|
||||
if err != nil {
|
||||
log.Println("Failed to load relays: " + err.Error())
|
||||
return nil
|
||||
@@ -532,7 +568,7 @@ func loadRelays(file string) []*relay {
|
||||
|
||||
var relays []*relay
|
||||
for _, line := range strings.Split(string(content), "\n") {
|
||||
if line == "" {
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -562,11 +598,11 @@ func saveRelays(file string, relays []*relay) error {
|
||||
for _, relay := range relays {
|
||||
content += relay.uri.String() + "\n"
|
||||
}
|
||||
return os.WriteFile(file, []byte(content), 0o777)
|
||||
return ioutil.WriteFile(file, []byte(content), 0777)
|
||||
}
|
||||
|
||||
func createTestCertificate() tls.Certificate {
|
||||
tmpDir, err := os.MkdirTemp("", "relaypoolsrv")
|
||||
tmpDir, err := ioutil.TempDir("", "relaypoolsrv")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@@ -621,42 +657,3 @@ func (lrw *loggingResponseWriter) WriteHeader(code int) {
|
||||
lrw.statusCode = code
|
||||
lrw.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
type errorTracker struct {
|
||||
errors *lru.TwoQueueCache[string, *errorCounter]
|
||||
}
|
||||
|
||||
type errorCounter struct {
|
||||
count atomic.Int32
|
||||
}
|
||||
|
||||
func newErrorTracker(size int) *errorTracker {
|
||||
cache, err := lru.New2Q[string, *errorCounter](size)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &errorTracker{
|
||||
errors: cache,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *errorTracker) AddError(host string) {
|
||||
entry, ok := b.errors.Get(host)
|
||||
if !ok {
|
||||
entry = &errorCounter{}
|
||||
b.errors.Add(host, entry)
|
||||
}
|
||||
c := entry.count.Add(1)
|
||||
log.Printf("Error count for %s is now %d", host, c)
|
||||
}
|
||||
|
||||
func (b *errorTracker) ClearErrors(host string) {
|
||||
b.errors.Remove(host)
|
||||
}
|
||||
|
||||
func (b *errorTracker) IsBlocked(host string) bool {
|
||||
if be, ok := b.errors.Get(host); ok {
|
||||
return be.count.Load() > 10
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -66,29 +65,3 @@ func TestHandleGetRequest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanonicalizeQueryValues(t *testing.T) {
|
||||
// This just demonstrates and validates the uri.Parse/String stuff in
|
||||
// regards to query strings.
|
||||
|
||||
in := "http://example.com/?some weird= query^value"
|
||||
exp := "http://example.com/?some+weird=+query%5Evalue"
|
||||
|
||||
uri, err := url.Parse(in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
str := uri.String()
|
||||
if str != in {
|
||||
// Just re-encoding the URL doesn't sanitize the query string.
|
||||
t.Errorf("expected %q, got %q", in, str)
|
||||
}
|
||||
|
||||
uri.RawQuery = uri.Query().Encode()
|
||||
str = uri.String()
|
||||
if str != exp {
|
||||
// The query string is now in correct format.
|
||||
t.Errorf("expected %q, got %q", exp, str)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,12 +10,11 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/syncthing/syncthing/lib/sync"
|
||||
)
|
||||
|
||||
func init() {
|
||||
processCollectorOpts := collectors.ProcessCollectorOpts{
|
||||
processCollectorOpts := prometheus.ProcessCollectorOpts{
|
||||
Namespace: "syncthing_relaypoolsrv",
|
||||
PidFn: func() (int, error) {
|
||||
return os.Getpid(), nil
|
||||
@@ -23,7 +22,7 @@ func init() {
|
||||
}
|
||||
|
||||
prometheus.MustRegister(
|
||||
collectors.NewProcessCollector(processCollectorOpts),
|
||||
prometheus.NewProcessCollector(processCollectorOpts),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -20,10 +20,10 @@ import (
|
||||
var (
|
||||
outboxesMut = sync.RWMutex{}
|
||||
outboxes = make(map[syncthingprotocol.DeviceID]chan interface{})
|
||||
numConnections atomic.Int64
|
||||
numConnections int64
|
||||
)
|
||||
|
||||
func listener(_, addr string, config *tls.Config, token string) {
|
||||
func listener(proto, addr string, config *tls.Config) {
|
||||
tcpListener, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
@@ -36,14 +36,8 @@ func listener(_, addr string, config *tls.Config, token string) {
|
||||
for {
|
||||
conn, isTLS, err := listener.AcceptNoWrapTLS()
|
||||
if err != nil {
|
||||
// Conn may be nil if accept failed, or non-nil if the initial
|
||||
// read to figure out if it's TLS or not failed. In the latter
|
||||
// case, close the connection before moving on.
|
||||
if conn != nil {
|
||||
conn.Close()
|
||||
}
|
||||
if debug {
|
||||
log.Println("Listener failed to accept:", err)
|
||||
log.Println("Listener failed to accept connection from", conn.RemoteAddr(), ". Possibly a TCP Ping.")
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -55,7 +49,7 @@ func listener(_, addr string, config *tls.Config, token string) {
|
||||
}
|
||||
|
||||
if isTLS {
|
||||
go protocolConnectionHandler(conn, config, token)
|
||||
go protocolConnectionHandler(conn, config)
|
||||
} else {
|
||||
go sessionConnectionHandler(conn)
|
||||
}
|
||||
@@ -63,7 +57,7 @@ func listener(_, addr string, config *tls.Config, token string) {
|
||||
}
|
||||
}
|
||||
|
||||
func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token string) {
|
||||
func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
|
||||
conn := tls.Server(tcpConn, config)
|
||||
if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil {
|
||||
if debug {
|
||||
@@ -82,7 +76,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token strin
|
||||
}
|
||||
|
||||
state := conn.ConnectionState()
|
||||
if debug && state.NegotiatedProtocol != protocol.ProtocolName {
|
||||
if (!state.NegotiatedProtocolIsMutual || state.NegotiatedProtocol != protocol.ProtocolName) && debug {
|
||||
log.Println("Protocol negotiation error")
|
||||
}
|
||||
|
||||
@@ -125,16 +119,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token strin
|
||||
|
||||
switch msg := message.(type) {
|
||||
case protocol.JoinRelayRequest:
|
||||
if token != "" && msg.Token != token {
|
||||
if debug {
|
||||
log.Printf("invalid token %s\n", msg.Token)
|
||||
}
|
||||
protocol.WriteMessage(conn, protocol.ResponseWrongToken)
|
||||
conn.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
if overLimit.Load() {
|
||||
if atomic.LoadInt32(&overLimit) > 0 {
|
||||
protocol.WriteMessage(conn, protocol.RelayFull{})
|
||||
if debug {
|
||||
log.Println("Refusing join request from", id, "due to being over limits")
|
||||
@@ -273,7 +258,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config, token strin
|
||||
conn.Close()
|
||||
}
|
||||
|
||||
if overLimit.Load() && !hasSessions(id) {
|
||||
if atomic.LoadInt32(&overLimit) > 0 && !hasSessions(id) {
|
||||
if debug {
|
||||
log.Println("Dropping", id, "as it has no sessions and we are over our limits")
|
||||
}
|
||||
@@ -366,8 +351,8 @@ func sessionConnectionHandler(conn net.Conn) {
|
||||
}
|
||||
|
||||
func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) {
|
||||
numConnections.Add(1)
|
||||
defer numConnections.Add(-1)
|
||||
atomic.AddInt64(&numConnections, 1)
|
||||
defer atomic.AddInt64(&numConnections, -1)
|
||||
|
||||
for {
|
||||
msg, err := protocol.ReadMessage(conn)
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
@@ -49,14 +50,13 @@ var (
|
||||
|
||||
sessionLimitBps int
|
||||
globalLimitBps int
|
||||
overLimit atomic.Bool
|
||||
overLimit int32
|
||||
descriptorLimit int64
|
||||
sessionLimiter *rate.Limiter
|
||||
globalLimiter *rate.Limiter
|
||||
networkBufferSize int
|
||||
|
||||
statusAddr string
|
||||
token string
|
||||
poolAddrs string
|
||||
pools []string
|
||||
providedBy string
|
||||
@@ -90,7 +90,6 @@ func main() {
|
||||
flag.IntVar(&globalLimitBps, "global-rate", globalLimitBps, "Global rate limit, in bytes/s")
|
||||
flag.BoolVar(&debug, "debug", debug, "Enable debug output")
|
||||
flag.StringVar(&statusAddr, "status-srv", ":22070", "Listen address for status service (blank to disable)")
|
||||
flag.StringVar(&token, "token", "", "Token to restrict access to the relay (optional). Disables joining any pools.")
|
||||
flag.StringVar(&poolAddrs, "pools", defaultPoolAddrs, "Comma separated list of relay pool addresses to join")
|
||||
flag.StringVar(&providedBy, "provided-by", "", "An optional description about who provides the relay")
|
||||
flag.StringVar(&extAddress, "ext-address", "", "An optional address to advertise as being available on.\n\tAllows listening on an unprivileged port with port forwarding from e.g. 443, and be connected to on port 443.")
|
||||
@@ -147,7 +146,7 @@ func main() {
|
||||
log.Println("Connection limit", descriptorLimit)
|
||||
|
||||
go monitorLimits()
|
||||
} else if err != nil && !build.IsWindows {
|
||||
} else if err != nil && runtime.GOOS != "windows" {
|
||||
log.Println("Assuming no connection limit, due to error retrieving rlimits:", err)
|
||||
}
|
||||
|
||||
@@ -194,22 +193,14 @@ func main() {
|
||||
cfg.Options.NATTimeoutS = natTimeout
|
||||
})
|
||||
natSvc := nat.NewService(id, wrapper)
|
||||
var ipVersion nat.IPVersion
|
||||
if strings.HasSuffix(proto, "4") {
|
||||
ipVersion = nat.IPv4Only
|
||||
} else if strings.HasSuffix(proto, "6") {
|
||||
ipVersion = nat.IPv6Only
|
||||
} else {
|
||||
ipVersion = nat.IPvAny
|
||||
}
|
||||
mapping := mapping{natSvc.NewMapping(nat.TCP, ipVersion, addr.IP, addr.Port)}
|
||||
mapping := mapping{natSvc.NewMapping(nat.TCP, addr.IP, addr.Port)}
|
||||
|
||||
if natEnabled {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go natSvc.Serve(ctx)
|
||||
defer cancel()
|
||||
found := make(chan struct{})
|
||||
mapping.OnChanged(func() {
|
||||
mapping.OnChanged(func(_ *nat.Mapping, _, _ []nat.Address) {
|
||||
select {
|
||||
case found <- struct{}{}:
|
||||
default:
|
||||
@@ -239,37 +230,14 @@ func main() {
|
||||
go statusService(statusAddr)
|
||||
}
|
||||
|
||||
uri, err := url.Parse(fmt.Sprintf("relay://%s/", mapping.Address()))
|
||||
uri, err := url.Parse(fmt.Sprintf("relay://%s/?id=%s&pingInterval=%s&networkTimeout=%s&sessionLimitBps=%d&globalLimitBps=%d&statusAddr=%s&providedBy=%s", mapping.Address(), id, pingInterval, networkTimeout, sessionLimitBps, globalLimitBps, statusAddr, providedBy))
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to construct URI", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Add properly encoded query string parameters to URL.
|
||||
query := make(url.Values)
|
||||
query.Set("id", id.String())
|
||||
query.Set("pingInterval", pingInterval.String())
|
||||
query.Set("networkTimeout", networkTimeout.String())
|
||||
if sessionLimitBps > 0 {
|
||||
query.Set("sessionLimitBps", fmt.Sprint(sessionLimitBps))
|
||||
}
|
||||
if globalLimitBps > 0 {
|
||||
query.Set("globalLimitBps", fmt.Sprint(globalLimitBps))
|
||||
}
|
||||
if statusAddr != "" {
|
||||
query.Set("statusAddr", statusAddr)
|
||||
}
|
||||
if providedBy != "" {
|
||||
query.Set("providedBy", providedBy)
|
||||
}
|
||||
uri.RawQuery = query.Encode()
|
||||
|
||||
log.Println("URI:", uri.String())
|
||||
|
||||
if token != "" {
|
||||
poolAddrs = ""
|
||||
}
|
||||
|
||||
if poolAddrs == defaultPoolAddrs {
|
||||
log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
|
||||
log.Println("!! Joining default relay pools, this relay will be available for public use. !!")
|
||||
@@ -285,7 +253,7 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
go listener(proto, listen, tlsCfg, token)
|
||||
go listener(proto, listen, tlsCfg)
|
||||
|
||||
sigs := make(chan os.Signal, 1)
|
||||
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
|
||||
@@ -316,10 +284,10 @@ func main() {
|
||||
func monitorLimits() {
|
||||
limitCheckTimer = time.NewTimer(time.Minute)
|
||||
for range limitCheckTimer.C {
|
||||
if numConnections.Load()+numProxies.Load() > descriptorLimit {
|
||||
overLimit.Store(true)
|
||||
if atomic.LoadInt64(&numConnections)+atomic.LoadInt64(&numProxies) > descriptorLimit {
|
||||
atomic.StoreInt32(&overLimit, 1)
|
||||
log.Println("Gone past our connection limits. Starting to refuse new/drop idle connections.")
|
||||
} else if overLimit.CompareAndSwap(true, false) {
|
||||
} else if atomic.CompareAndSwapInt32(&overLimit, 1, 0) {
|
||||
log.Println("Dropped below our connection limits. Accepting new connections.")
|
||||
}
|
||||
limitCheckTimer.Reset(time.Minute)
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -56,7 +56,7 @@ func poolHandler(pool string, uri *url.URL, mapping mapping, ownCert tls.Certifi
|
||||
continue
|
||||
}
|
||||
|
||||
bs, err := io.ReadAll(resp.Body)
|
||||
bs, err := ioutil.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
log.Printf("Error joining pool %v: reading response: %v", pool, err)
|
||||
@@ -74,8 +74,9 @@ func poolHandler(pool string, uri *url.URL, mapping mapping, ownCert tls.Certifi
|
||||
log.Printf("Joined pool %s, rejoining in %v", pool, rejoin)
|
||||
time.Sleep(rejoin)
|
||||
continue
|
||||
} else {
|
||||
log.Printf("Joined pool %s, failed to deserialize response: %v", pool, err)
|
||||
}
|
||||
log.Printf("Joined pool %s, failed to deserialize response: %v", pool, err)
|
||||
|
||||
case http.StatusInternalServerError:
|
||||
log.Printf("Failed to join %v: server error", pool)
|
||||
|
||||
@@ -23,8 +23,8 @@ var (
|
||||
sessionMut = sync.RWMutex{}
|
||||
activeSessions = make([]*session, 0)
|
||||
pendingSessions = make(map[string]*session)
|
||||
numProxies atomic.Int64
|
||||
bytesProxied atomic.Int64
|
||||
numProxies int64
|
||||
bytesProxied int64
|
||||
)
|
||||
|
||||
func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionRateLimit, globalRateLimit *rate.Limiter) *session {
|
||||
@@ -251,8 +251,8 @@ func (s *session) proxy(c1, c2 net.Conn) error {
|
||||
log.Println("Proxy", c1.RemoteAddr(), "->", c2.RemoteAddr())
|
||||
}
|
||||
|
||||
numProxies.Add(1)
|
||||
defer numProxies.Add(-1)
|
||||
atomic.AddInt64(&numProxies, 1)
|
||||
defer atomic.AddInt64(&numProxies, -1)
|
||||
|
||||
buf := make([]byte, networkBufferSize)
|
||||
for {
|
||||
@@ -262,7 +262,7 @@ func (s *session) proxy(c1, c2 net.Conn) error {
|
||||
return err
|
||||
}
|
||||
|
||||
bytesProxied.Add(int64(n))
|
||||
atomic.AddInt64(&bytesProxied, int64(n))
|
||||
|
||||
if debug {
|
||||
log.Printf("%d bytes from %s to %s", n, c1.RemoteAddr(), c2.RemoteAddr())
|
||||
|
||||
@@ -36,7 +36,7 @@ func statusService(addr string) {
|
||||
}
|
||||
}
|
||||
|
||||
func getStatus(w http.ResponseWriter, _ *http.Request) {
|
||||
func getStatus(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
status := make(map[string]interface{})
|
||||
|
||||
@@ -51,9 +51,9 @@ func getStatus(w http.ResponseWriter, _ *http.Request) {
|
||||
status["numPendingSessionKeys"] = len(pendingSessions)
|
||||
status["numActiveSessions"] = len(activeSessions)
|
||||
sessionMut.Unlock()
|
||||
status["numConnections"] = numConnections.Load()
|
||||
status["numProxies"] = numProxies.Load()
|
||||
status["bytesProxied"] = bytesProxied.Load()
|
||||
status["numConnections"] = atomic.LoadInt64(&numConnections)
|
||||
status["numProxies"] = atomic.LoadInt64(&numProxies)
|
||||
status["bytesProxied"] = atomic.LoadInt64(&bytesProxied)
|
||||
status["goVersion"] = runtime.Version()
|
||||
status["goOS"] = runtime.GOOS
|
||||
status["goArch"] = runtime.GOARCH
|
||||
@@ -88,13 +88,13 @@ func getStatus(w http.ResponseWriter, _ *http.Request) {
|
||||
}
|
||||
|
||||
type rateCalculator struct {
|
||||
counter *atomic.Int64
|
||||
counter *int64 // atomic, must remain 64-bit aligned
|
||||
rates []int64
|
||||
prev int64
|
||||
startTime time.Time
|
||||
}
|
||||
|
||||
func newRateCalculator(keepIntervals int, interval time.Duration, counter *atomic.Int64) *rateCalculator {
|
||||
func newRateCalculator(keepIntervals int, interval time.Duration, counter *int64) *rateCalculator {
|
||||
r := &rateCalculator{
|
||||
rates: make([]int64, keepIntervals),
|
||||
counter: counter,
|
||||
@@ -112,7 +112,7 @@ func (r *rateCalculator) updateRates(interval time.Duration) {
|
||||
next := now.Truncate(interval).Add(interval)
|
||||
time.Sleep(next.Sub(now))
|
||||
|
||||
cur := r.counter.Load()
|
||||
cur := atomic.LoadInt64(r.counter)
|
||||
rate := int64(float64(cur-r.prev) / interval.Seconds())
|
||||
copy(r.rates[1:], r.rates)
|
||||
r.rates[0] = rate
|
||||
|
||||
@@ -127,6 +127,10 @@ func stdinReader(c chan<- string) {
|
||||
}
|
||||
|
||||
func connectToStdio(stdin <-chan string, conn net.Conn) {
|
||||
go func() {
|
||||
|
||||
}()
|
||||
|
||||
buf := make([]byte, 1024)
|
||||
for {
|
||||
conn.SetReadDeadline(time.Now().Add(time.Millisecond))
|
||||
|
||||
@@ -9,6 +9,7 @@ package main
|
||||
import (
|
||||
"flag"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
@@ -68,7 +69,7 @@ func gen() {
|
||||
}
|
||||
|
||||
func sign(keyname, dataname string) {
|
||||
privkey, err := os.ReadFile(keyname)
|
||||
privkey, err := ioutil.ReadFile(keyname)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@@ -94,7 +95,7 @@ func sign(keyname, dataname string) {
|
||||
}
|
||||
|
||||
func verifyWithFile(signame, dataname, keyname string) {
|
||||
pubkey, err := os.ReadFile(keyname)
|
||||
pubkey, err := ioutil.ReadFile(keyname)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@@ -102,7 +103,7 @@ func verifyWithFile(signame, dataname, keyname string) {
|
||||
}
|
||||
|
||||
func verifyWithKey(signame, dataname string, pubkey []byte) {
|
||||
sig, err := os.ReadFile(signame)
|
||||
sig, err := ioutil.ReadFile(signame)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -7,122 +7,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"flag"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/syncthing/syncthing/lib/httpcache"
|
||||
"github.com/syncthing/syncthing/lib/upgrade"
|
||||
)
|
||||
|
||||
type cli struct {
|
||||
Listen string `default:":8080" help:"Listen address"`
|
||||
URL string `short:"u" default:"https://api.github.com/repos/syncthing/syncthing/releases?per_page=25" help:"GitHub releases url"`
|
||||
Forward []string `short:"f" help:"Forwarded pages, format: /path->https://example/com/url"`
|
||||
CacheTime time.Duration `default:"15m" help:"Cache time"`
|
||||
}
|
||||
const defaultURL = "https://api.github.com/repos/syncthing/syncthing/releases?per_page=25"
|
||||
|
||||
func main() {
|
||||
var params cli
|
||||
kong.Parse(¶ms)
|
||||
if err := server(¶ms); err != nil {
|
||||
fmt.Printf("Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
url := flag.String("u", defaultURL, "GitHub releases url")
|
||||
flag.Parse()
|
||||
|
||||
func server(params *cli) error {
|
||||
http.Handle("/meta.json", httpcache.SinglePath(&githubReleases{url: params.URL}, params.CacheTime))
|
||||
|
||||
for _, fwd := range params.Forward {
|
||||
path, url, ok := strings.Cut(fwd, "->")
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid forward: %q", fwd)
|
||||
}
|
||||
http.Handle(path, httpcache.SinglePath(&proxy{url: url}, params.CacheTime))
|
||||
}
|
||||
|
||||
return http.ListenAndServe(params.Listen, nil)
|
||||
}
|
||||
|
||||
type githubReleases struct {
|
||||
url string
|
||||
}
|
||||
|
||||
func (p *githubReleases) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
|
||||
log.Println("Fetching", p.url)
|
||||
rels := upgrade.FetchLatestReleases(p.url, "")
|
||||
rels := upgrade.FetchLatestReleases(*url, "")
|
||||
if rels == nil {
|
||||
http.Error(w, "no releases", http.StatusInternalServerError)
|
||||
return
|
||||
// An error was already logged
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
sort.Sort(upgrade.SortByRelease(rels))
|
||||
rels = filterForLatest(rels)
|
||||
|
||||
// Move the URL used for browser downloads to the URL field, and remove
|
||||
// the browser URL field. This avoids going via the GitHub API for
|
||||
// downloads, since Syncthing uses the URL field.
|
||||
for _, rel := range rels {
|
||||
for j, asset := range rel.Assets {
|
||||
rel.Assets[j].URL = asset.BrowserURL
|
||||
rel.Assets[j].BrowserURL = ""
|
||||
}
|
||||
}
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
_ = json.NewEncoder(buf).Encode(rels)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET")
|
||||
w.Write(buf.Bytes())
|
||||
}
|
||||
|
||||
type proxy struct {
|
||||
url string
|
||||
}
|
||||
|
||||
func (p *proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
log.Println("Fetching", p.url)
|
||||
req, err := http.NewRequestWithContext(req.Context(), http.MethodGet, p.url, nil)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
ct := resp.Header.Get("Content-Type")
|
||||
w.Header().Set("Content-Type", ct)
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
w.Header().Set("Cache-Control", "public, max-age=900")
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET")
|
||||
}
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
if strings.HasPrefix(ct, "application/json") {
|
||||
// Special JSON handling; clean it up a bit.
|
||||
var v interface{}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&v); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
} else {
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
if err := json.NewEncoder(os.Stdout).Encode(rels); err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ func main() {
|
||||
|
||||
found := make(chan result)
|
||||
stop := make(chan struct{})
|
||||
var count atomic.Int64
|
||||
var count int64
|
||||
|
||||
// Print periodic progress reports.
|
||||
go printProgress(prefix, &count)
|
||||
@@ -72,7 +72,7 @@ func main() {
|
||||
// Try certificates until one is found that has the prefix at the start of
|
||||
// the resulting device ID. Increments count atomically, sends the result to
|
||||
// found, returns when stop is closed.
|
||||
func generatePrefixed(prefix string, count *atomic.Int64, found chan<- result, stop <-chan struct{}) {
|
||||
func generatePrefixed(prefix string, count *int64, found chan<- result, stop <-chan struct{}) {
|
||||
notBefore := time.Now()
|
||||
notAfter := time.Date(2049, 12, 31, 23, 59, 59, 0, time.UTC)
|
||||
|
||||
@@ -109,7 +109,7 @@ func generatePrefixed(prefix string, count *atomic.Int64, found chan<- result, s
|
||||
}
|
||||
|
||||
id := protocol.NewDeviceID(derBytes)
|
||||
count.Add(1)
|
||||
atomic.AddInt64(count, 1)
|
||||
|
||||
if strings.HasPrefix(id.String(), prefix) {
|
||||
select {
|
||||
@@ -121,7 +121,7 @@ func generatePrefixed(prefix string, count *atomic.Int64, found chan<- result, s
|
||||
}
|
||||
}
|
||||
|
||||
func printProgress(prefix string, count *atomic.Int64) {
|
||||
func printProgress(prefix string, count *int64) {
|
||||
started := time.Now()
|
||||
wantBits := 5 * len(prefix)
|
||||
if wantBits > 63 {
|
||||
@@ -132,7 +132,7 @@ func printProgress(prefix string, count *atomic.Int64) {
|
||||
fmt.Printf("Want %d bits for prefix %q, about %.2g certs to test (statistically speaking)\n", wantBits, prefix, expectedIterations)
|
||||
|
||||
for range time.NewTicker(15 * time.Second).C {
|
||||
tried := count.Load()
|
||||
tried := atomic.LoadInt64(count)
|
||||
elapsed := time.Since(started)
|
||||
rate := float64(tried) / elapsed.Seconds()
|
||||
expected := timeStr(expectedIterations / rate)
|
||||
|
||||
@@ -10,10 +10,8 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -27,7 +25,6 @@ import (
|
||||
type APIClient interface {
|
||||
Get(url string) (*http.Response, error)
|
||||
Post(url, body string) (*http.Response, error)
|
||||
PutJSON(url string, o interface{}) (*http.Response, error)
|
||||
}
|
||||
|
||||
type apiClient struct {
|
||||
@@ -121,36 +118,20 @@ func (c *apiClient) Do(req *http.Request) (*http.Response, error) {
|
||||
return resp, checkResponse(resp)
|
||||
}
|
||||
|
||||
func (c *apiClient) Request(url, method string, r io.Reader) (*http.Response, error) {
|
||||
request, err := http.NewRequest(method, c.Endpoint()+"rest/"+url, r)
|
||||
func (c *apiClient) Get(url string) (*http.Response, error) {
|
||||
request, err := http.NewRequest("GET", c.Endpoint()+"rest/"+url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.Do(request)
|
||||
}
|
||||
|
||||
func (c *apiClient) RequestString(url, method, data string) (*http.Response, error) {
|
||||
return c.Request(url, method, bytes.NewBufferString(data))
|
||||
}
|
||||
|
||||
func (c *apiClient) RequestJSON(url, method string, o interface{}) (*http.Response, error) {
|
||||
data, err := json.Marshal(o)
|
||||
func (c *apiClient) Post(url, body string) (*http.Response, error) {
|
||||
request, err := http.NewRequest("POST", c.Endpoint()+"rest/"+url, bytes.NewBufferString(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.Request(url, method, bytes.NewBuffer(data))
|
||||
}
|
||||
|
||||
func (c *apiClient) Get(url string) (*http.Response, error) {
|
||||
return c.RequestString(url, "GET", "")
|
||||
}
|
||||
|
||||
func (c *apiClient) Post(url, body string) (*http.Response, error) {
|
||||
return c.RequestString(url, "POST", body)
|
||||
}
|
||||
|
||||
func (c *apiClient) PutJSON(url string, o interface{}) (*http.Response, error) {
|
||||
return c.RequestJSON(url, "PUT", o)
|
||||
return c.Do(request)
|
||||
}
|
||||
|
||||
var errNotFound = errors.New("invalid endpoint or API call")
|
||||
|
||||
@@ -8,12 +8,11 @@ package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/AudriusButkevicius/recli"
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
@@ -24,20 +23,9 @@ type configHandler struct {
|
||||
err error
|
||||
}
|
||||
|
||||
type configCommand struct {
|
||||
Args []string `arg:"" default:"-h"`
|
||||
}
|
||||
|
||||
func (c *configCommand) Run(ctx Context, _ *kong.Context) error {
|
||||
app := cli.NewApp()
|
||||
app.Name = "syncthing"
|
||||
app.Author = "The Syncthing Authors"
|
||||
app.Metadata = map[string]interface{}{
|
||||
"clientFactory": ctx.clientFactory,
|
||||
}
|
||||
|
||||
func getConfigCommand(f *apiClientFactory) (cli.Command, error) {
|
||||
h := new(configHandler)
|
||||
h.client, h.err = ctx.clientFactory.getClient()
|
||||
h.client, h.err = f.getClient()
|
||||
if h.err == nil {
|
||||
h.cfg, h.err = getConfig(h.client)
|
||||
}
|
||||
@@ -50,15 +38,17 @@ func (c *configCommand) Run(ctx Context, _ *kong.Context) error {
|
||||
|
||||
commands, err := recli.New(recliCfg).Construct(&h.cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("config reflect: %w", err)
|
||||
return cli.Command{}, fmt.Errorf("config reflect: %w", err)
|
||||
}
|
||||
|
||||
app.Commands = commands
|
||||
app.HideHelp = true
|
||||
app.Before = h.configBefore
|
||||
app.After = h.configAfter
|
||||
|
||||
return app.Run(append([]string{app.Name}, c.Args...))
|
||||
return cli.Command{
|
||||
Name: "config",
|
||||
HideHelp: true,
|
||||
Usage: "Configuration modification command group",
|
||||
Subcommands: commands,
|
||||
Before: h.configBefore,
|
||||
After: h.configAfter,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (h *configHandler) configBefore(c *cli.Context) error {
|
||||
@@ -70,7 +60,7 @@ func (h *configHandler) configBefore(c *cli.Context) error {
|
||||
return h.err
|
||||
}
|
||||
|
||||
func (h *configHandler) configAfter(_ *cli.Context) error {
|
||||
func (h *configHandler) configAfter(c *cli.Context) error {
|
||||
if h.err != nil {
|
||||
// Error was already returned in configBefore
|
||||
return nil
|
||||
|
||||
@@ -9,37 +9,47 @@ package cli
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
type fileCommand struct {
|
||||
FolderID string `arg:""`
|
||||
Path string `arg:""`
|
||||
var debugCommand = cli.Command{
|
||||
Name: "debug",
|
||||
HideHelp: true,
|
||||
Usage: "Debug command group",
|
||||
Subcommands: []cli.Command{
|
||||
{
|
||||
Name: "file",
|
||||
Usage: "Show information about a file (or directory/symlink)",
|
||||
ArgsUsage: "FOLDER-ID PATH",
|
||||
Action: expects(2, debugFile()),
|
||||
},
|
||||
indexCommand,
|
||||
{
|
||||
Name: "profile",
|
||||
Usage: "Save a profile to help figuring out what Syncthing does.",
|
||||
ArgsUsage: "cpu | heap",
|
||||
Action: expects(1, profile()),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func (f *fileCommand) Run(ctx Context) error {
|
||||
indexDumpOutput := indexDumpOutputWrapper(ctx.clientFactory)
|
||||
|
||||
query := make(url.Values)
|
||||
query.Set("folder", f.FolderID)
|
||||
query.Set("file", normalizePath(f.Path))
|
||||
return indexDumpOutput("debug/file?" + query.Encode())
|
||||
}
|
||||
|
||||
type profileCommand struct {
|
||||
Type string `arg:"" help:"cpu | heap"`
|
||||
}
|
||||
|
||||
func (p *profileCommand) Run(ctx Context) error {
|
||||
switch t := p.Type; t {
|
||||
case "cpu", "heap":
|
||||
return saveToFile(fmt.Sprintf("debug/%vprof", p.Type), ctx.clientFactory)
|
||||
default:
|
||||
return fmt.Errorf("expected cpu or heap as argument, got %v", t)
|
||||
func debugFile() cli.ActionFunc {
|
||||
return func(c *cli.Context) error {
|
||||
query := make(url.Values)
|
||||
query.Set("folder", c.Args()[0])
|
||||
query.Set("file", normalizePath(c.Args()[1]))
|
||||
return indexDumpOutput("debug/file?" + query.Encode())(c)
|
||||
}
|
||||
}
|
||||
|
||||
type debugCommand struct {
|
||||
File fileCommand `cmd:"" help:"Show information about a file (or directory/symlink)"`
|
||||
Profile profileCommand `cmd:"" help:"Save a profile to help figuring out what Syncthing does"`
|
||||
Index indexCommand `cmd:"" help:"Show information about the index (database)"`
|
||||
func profile() cli.ActionFunc {
|
||||
return func(c *cli.Context) error {
|
||||
switch t := c.Args()[0]; t {
|
||||
case "cpu", "heap":
|
||||
return saveToFile(fmt.Sprintf("debug/%vprof", c.Args()[0]))(c)
|
||||
default:
|
||||
return fmt.Errorf("expected cpu or heap as argument, got %v", t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,25 +11,36 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
type errorsCommand struct {
|
||||
Show struct{} `cmd:"" help:"Show pending errors"`
|
||||
Push errorsPushCommand `cmd:"" help:"Push an error to active clients"`
|
||||
Clear struct{} `cmd:"" help:"Clear pending errors"`
|
||||
var errorsCommand = cli.Command{
|
||||
Name: "errors",
|
||||
HideHelp: true,
|
||||
Usage: "Error command group",
|
||||
Subcommands: []cli.Command{
|
||||
{
|
||||
Name: "show",
|
||||
Usage: "Show pending errors",
|
||||
Action: expects(0, indexDumpOutput("system/error")),
|
||||
},
|
||||
{
|
||||
Name: "push",
|
||||
Usage: "Push an error to active clients",
|
||||
ArgsUsage: "[error message]",
|
||||
Action: expects(1, errorsPush),
|
||||
},
|
||||
{
|
||||
Name: "clear",
|
||||
Usage: "Clear pending errors",
|
||||
Action: expects(0, emptyPost("system/error/clear")),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
type errorsPushCommand struct {
|
||||
ErrorMessage string `arg:""`
|
||||
}
|
||||
|
||||
func (e *errorsPushCommand) Run(ctx Context) error {
|
||||
client, err := ctx.clientFactory.getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
errStr := e.ErrorMessage
|
||||
func errorsPush(c *cli.Context) error {
|
||||
client := c.App.Metadata["client"].(APIClient)
|
||||
errStr := strings.Join(c.Args(), " ")
|
||||
response, err := client.Post("system/error", strings.TrimSpace(errStr))
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -48,13 +59,3 @@ func (e *errorsPushCommand) Run(ctx Context) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (*errorsCommand) Run(ctx Context, kongCtx *kong.Context) error {
|
||||
switch kongCtx.Selected().Name {
|
||||
case "show":
|
||||
return indexDumpOutput("system/error", ctx.clientFactory)
|
||||
case "clear":
|
||||
return emptyPost("system/error/clear", ctx.clientFactory)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -7,26 +7,32 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
type indexCommand struct {
|
||||
Dump struct{} `cmd:"" help:"Print the entire db"`
|
||||
DumpSize struct{} `cmd:"" help:"Print the db size of different categories of information"`
|
||||
Check struct{} `cmd:"" help:"Check the database for inconsistencies"`
|
||||
Account struct{} `cmd:"" help:"Print key and value size statistics per key type"`
|
||||
}
|
||||
|
||||
func (*indexCommand) Run(kongCtx *kong.Context) error {
|
||||
switch kongCtx.Selected().Name {
|
||||
case "dump":
|
||||
return indexDump()
|
||||
case "dump-size":
|
||||
return indexDumpSize()
|
||||
case "check":
|
||||
return indexCheck()
|
||||
case "account":
|
||||
return indexAccount()
|
||||
}
|
||||
return nil
|
||||
var indexCommand = cli.Command{
|
||||
Name: "index",
|
||||
Usage: "Show information about the index (database)",
|
||||
Subcommands: []cli.Command{
|
||||
{
|
||||
Name: "dump",
|
||||
Usage: "Print the entire db",
|
||||
Action: expects(0, indexDump),
|
||||
},
|
||||
{
|
||||
Name: "dump-size",
|
||||
Usage: "Print the db size of different categories of information",
|
||||
Action: expects(0, indexDumpSize),
|
||||
},
|
||||
{
|
||||
Name: "check",
|
||||
Usage: "Check the database for inconsistencies",
|
||||
Action: expects(0, indexCheck),
|
||||
},
|
||||
{
|
||||
Name: "account",
|
||||
Usage: "Print key and value size statistics per key type",
|
||||
Action: expects(0, indexAccount),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -10,10 +10,12 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
// indexAccount prints key and data size statistics per class
|
||||
func indexAccount() error {
|
||||
func indexAccount(*cli.Context) error {
|
||||
ldb, err := getDB()
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -11,11 +11,13 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/db"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
)
|
||||
|
||||
func indexDump() error {
|
||||
func indexDump(*cli.Context) error {
|
||||
ldb, err := getDB()
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -7,33 +7,53 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/db"
|
||||
)
|
||||
|
||||
func indexDumpSize() error {
|
||||
type sizedElement struct {
|
||||
key string
|
||||
size int
|
||||
}
|
||||
type SizedElement struct {
|
||||
key string
|
||||
size int
|
||||
}
|
||||
|
||||
type ElementHeap []SizedElement
|
||||
|
||||
func (h ElementHeap) Len() int { return len(h) }
|
||||
func (h ElementHeap) Less(i, j int) bool { return h[i].size > h[j].size }
|
||||
func (h ElementHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
|
||||
|
||||
func (h *ElementHeap) Push(x interface{}) {
|
||||
*h = append(*h, x.(SizedElement))
|
||||
}
|
||||
|
||||
func (h *ElementHeap) Pop() interface{} {
|
||||
old := *h
|
||||
n := len(old)
|
||||
x := old[n-1]
|
||||
*h = old[0 : n-1]
|
||||
return x
|
||||
}
|
||||
|
||||
func indexDumpSize(*cli.Context) error {
|
||||
ldb, err := getDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
h := &ElementHeap{}
|
||||
heap.Init(h)
|
||||
|
||||
it, err := ldb.NewPrefixIterator(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var elems []sizedElement
|
||||
var ele SizedElement
|
||||
for it.Next() {
|
||||
var ele sizedElement
|
||||
|
||||
key := it.Key()
|
||||
switch key[0] {
|
||||
case db.KeyTypeDevice:
|
||||
@@ -74,13 +94,11 @@ func indexDumpSize() error {
|
||||
ele.key = fmt.Sprintf("UNKNOWN:%x", key)
|
||||
}
|
||||
ele.size = len(it.Value())
|
||||
elems = append(elems, ele)
|
||||
heap.Push(h, ele)
|
||||
}
|
||||
|
||||
sort.Slice(elems, func(i, j int) bool {
|
||||
return elems[i].size > elems[j].size
|
||||
})
|
||||
for _, ele := range elems {
|
||||
for h.Len() > 0 {
|
||||
ele = heap.Pop(h).(SizedElement)
|
||||
fmt.Println(ele.key, ele.size)
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,8 @@ import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/db"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
)
|
||||
@@ -33,7 +35,7 @@ type sequenceKey struct {
|
||||
sequence uint64
|
||||
}
|
||||
|
||||
func indexCheck() (err error) {
|
||||
func indexCheck(*cli.Context) (err error) {
|
||||
ldb, err := getDB()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -55,7 +57,7 @@ func indexCheck() (err error) {
|
||||
defer func() {
|
||||
if err == nil {
|
||||
if success {
|
||||
fmt.Println("Index check completed successfully.")
|
||||
fmt.Println("Index check completed succesfully.")
|
||||
} else {
|
||||
err = errors.New("Inconsistencies found in the index")
|
||||
}
|
||||
@@ -347,7 +349,7 @@ func indexCheck() (err error) {
|
||||
|
||||
func needsLocally(vl db.VersionList) bool {
|
||||
gfv, gok := vl.GetGlobal()
|
||||
if !gok { // That's weird, but we hardly need something non-existent
|
||||
if !gok { // That's weird, but we hardly need something non-existant
|
||||
return false
|
||||
}
|
||||
fv, ok := vl.Get(protocol.LocalDeviceID[:])
|
||||
|
||||
@@ -9,87 +9,152 @@ package cli
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/flynn-archive/go-shlex"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/urfave/cli"
|
||||
|
||||
"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
)
|
||||
|
||||
type CLI struct {
|
||||
cmdutil.CommonOptions
|
||||
DataDir string `name:"data" placeholder:"PATH" env:"STDATADIR" help:"Set data directory (database and logs)"`
|
||||
type preCli struct {
|
||||
GUIAddress string `name:"gui-address"`
|
||||
GUIAPIKey string `name:"gui-apikey"`
|
||||
|
||||
Show showCommand `cmd:"" help:"Show command group"`
|
||||
Debug debugCommand `cmd:"" help:"Debug command group"`
|
||||
Operations operationCommand `cmd:"" help:"Operation command group"`
|
||||
Errors errorsCommand `cmd:"" help:"Error command group"`
|
||||
Config configCommand `cmd:"" help:"Configuration modification command group" passthrough:""`
|
||||
Stdin stdinCommand `cmd:"" name:"-" help:"Read commands from stdin"`
|
||||
HomeDir string `name:"home"`
|
||||
ConfDir string `name:"config"`
|
||||
DataDir string `name:"data"`
|
||||
}
|
||||
|
||||
type Context struct {
|
||||
clientFactory *apiClientFactory
|
||||
}
|
||||
func Run() error {
|
||||
// This is somewhat a hack around a chicken and egg problem. We need to set
|
||||
// the home directory and potentially other flags to know where the
|
||||
// syncthing instance is running in order to get it's config ... which we
|
||||
// then use to construct the actual CLI ... at which point it's too late to
|
||||
// add flags there...
|
||||
c := preCli{}
|
||||
parseFlags(&c)
|
||||
|
||||
func (cli CLI) AfterApply(kongCtx *kong.Context) error {
|
||||
err := cmdutil.SetConfigDataLocationsFromFlags(cli.HomeDir, cli.ConfDir, cli.DataDir)
|
||||
// Not set as default above because the strings can be really long.
|
||||
err := cmdutil.SetConfigDataLocationsFromFlags(c.HomeDir, c.ConfDir, c.DataDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("command line options: %w", err)
|
||||
return errors.Wrap(err, "Command line options:")
|
||||
}
|
||||
|
||||
clientFactory := &apiClientFactory{
|
||||
cfg: config.GUIConfiguration{
|
||||
RawAddress: cli.GUIAddress,
|
||||
APIKey: cli.GUIAPIKey,
|
||||
RawAddress: c.GUIAddress,
|
||||
APIKey: c.GUIAPIKey,
|
||||
},
|
||||
}
|
||||
|
||||
context := Context{
|
||||
clientFactory: clientFactory,
|
||||
configCommand, err := getConfigCommand(clientFactory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
kongCtx.Bind(context)
|
||||
return nil
|
||||
// Implement the same flags at the upper CLI, but do nothing with them.
|
||||
// This is so that the usage text is the same
|
||||
fakeFlags := []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "gui-address",
|
||||
Value: "URL",
|
||||
Usage: "Override GUI address (e.g. \"http://192.0.2.42:8443\")",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "gui-apikey",
|
||||
Value: "API-KEY",
|
||||
Usage: "Override GUI API key",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "home",
|
||||
Value: "PATH",
|
||||
Usage: "Set configuration and data directory",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "conf",
|
||||
Value: "PATH",
|
||||
Usage: "Set configuration directory (config and keys)",
|
||||
},
|
||||
}
|
||||
|
||||
// Construct the actual CLI
|
||||
app := cli.NewApp()
|
||||
app.Author = "The Syncthing Authors"
|
||||
app.Metadata = map[string]interface{}{
|
||||
"clientFactory": clientFactory,
|
||||
}
|
||||
app.Commands = []cli.Command{{
|
||||
Name: "cli",
|
||||
Usage: "Syncthing command line interface",
|
||||
Flags: fakeFlags,
|
||||
Subcommands: []cli.Command{
|
||||
configCommand,
|
||||
showCommand,
|
||||
operationCommand,
|
||||
errorsCommand,
|
||||
debugCommand,
|
||||
{
|
||||
Name: "-",
|
||||
HideHelp: true,
|
||||
Usage: "Read commands from stdin",
|
||||
Action: func(ctx *cli.Context) error {
|
||||
if ctx.NArg() > 0 {
|
||||
return errors.New("command does not expect any arguments")
|
||||
}
|
||||
|
||||
// Drop the `-` not to recurse into self.
|
||||
args := make([]string, len(os.Args)-1)
|
||||
copy(args, os.Args)
|
||||
|
||||
fmt.Println("Reading commands from stdin...", args)
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
for scanner.Scan() {
|
||||
input, err := shlex.Split(scanner.Text())
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "parsing input")
|
||||
}
|
||||
if len(input) == 0 {
|
||||
continue
|
||||
}
|
||||
err = app.Run(append(args, input...))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return scanner.Err()
|
||||
},
|
||||
},
|
||||
},
|
||||
}}
|
||||
|
||||
return app.Run(os.Args)
|
||||
}
|
||||
|
||||
type stdinCommand struct{}
|
||||
|
||||
func (*stdinCommand) Run() error {
|
||||
// Drop the `-` not to recurse into self.
|
||||
args := make([]string, len(os.Args)-1)
|
||||
copy(args, os.Args)
|
||||
|
||||
fmt.Println("Reading commands from stdin...", args)
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
for scanner.Scan() {
|
||||
input, err := shlex.Split(scanner.Text())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing input: %w", err)
|
||||
func parseFlags(c *preCli) error {
|
||||
// kong only needs to parse the global arguments after "cli" and before the
|
||||
// subcommand (if any).
|
||||
if len(os.Args) <= 2 {
|
||||
return nil
|
||||
}
|
||||
args := os.Args[2:]
|
||||
for i := 0; i < len(args); i++ {
|
||||
if !strings.HasPrefix(args[i], "--") {
|
||||
args = args[:i]
|
||||
break
|
||||
}
|
||||
if len(input) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var cli CLI
|
||||
p, err := kong.New(&cli)
|
||||
if err != nil {
|
||||
// can't happen, really
|
||||
return fmt.Errorf("creating parser: %w", err)
|
||||
}
|
||||
ctx, err := p.Parse(input)
|
||||
if err != nil {
|
||||
fmt.Println("Error:", err)
|
||||
continue
|
||||
}
|
||||
if err := ctx.Run(); err != nil {
|
||||
fmt.Println("Error:", err)
|
||||
continue
|
||||
if !strings.Contains(args[i], "=") {
|
||||
i++
|
||||
}
|
||||
}
|
||||
return scanner.Err()
|
||||
// We don't want kong to print anything nor os.Exit (e.g. on -h)
|
||||
parser, err := kong.New(c, kong.Writers(ioutil.Discard, ioutil.Discard), kong.Exit(func(int) {}))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = parser.Parse(args)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -7,48 +7,42 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
type folderOverrideCommand struct {
|
||||
FolderID string `arg:""`
|
||||
var operationCommand = cli.Command{
|
||||
Name: "operations",
|
||||
HideHelp: true,
|
||||
Usage: "Operation command group",
|
||||
Subcommands: []cli.Command{
|
||||
{
|
||||
Name: "restart",
|
||||
Usage: "Restart syncthing",
|
||||
Action: expects(0, emptyPost("system/restart")),
|
||||
},
|
||||
{
|
||||
Name: "shutdown",
|
||||
Usage: "Shutdown syncthing",
|
||||
Action: expects(0, emptyPost("system/shutdown")),
|
||||
},
|
||||
{
|
||||
Name: "upgrade",
|
||||
Usage: "Upgrade syncthing (if a newer version is available)",
|
||||
Action: expects(0, emptyPost("system/upgrade")),
|
||||
},
|
||||
{
|
||||
Name: "folder-override",
|
||||
Usage: "Override changes on folder (remote for sendonly, local for receiveonly). WARNING: Destructive - deletes/changes your data.",
|
||||
ArgsUsage: "[folder id]",
|
||||
Action: expects(1, foldersOverride),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
type defaultIgnoresCommand struct {
|
||||
Path string `arg:""`
|
||||
}
|
||||
|
||||
type operationCommand struct {
|
||||
Restart struct{} `cmd:"" help:"Restart syncthing"`
|
||||
Shutdown struct{} `cmd:"" help:"Shutdown syncthing"`
|
||||
Upgrade struct{} `cmd:"" help:"Upgrade syncthing (if a newer version is available)"`
|
||||
FolderOverride folderOverrideCommand `cmd:"" help:"Override changes on folder (remote for sendonly, local for receiveonly). WARNING: Destructive - deletes/changes your data"`
|
||||
DefaultIgnores defaultIgnoresCommand `cmd:"" help:"Set the default ignores (config) from a file"`
|
||||
}
|
||||
|
||||
func (*operationCommand) Run(ctx Context, kongCtx *kong.Context) error {
|
||||
f := ctx.clientFactory
|
||||
|
||||
switch kongCtx.Selected().Name {
|
||||
case "restart":
|
||||
return emptyPost("system/restart", f)
|
||||
case "shutdown":
|
||||
return emptyPost("system/shutdown", f)
|
||||
case "upgrade":
|
||||
return emptyPost("system/upgrade", f)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *folderOverrideCommand) Run(ctx Context) error {
|
||||
client, err := ctx.clientFactory.getClient()
|
||||
func foldersOverride(c *cli.Context) error {
|
||||
client, err := getClientFactory(c).getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -56,7 +50,7 @@ func (f *folderOverrideCommand) Run(ctx Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rid := f.FolderID
|
||||
rid := c.Args()[0]
|
||||
for _, folder := range cfg.Folders {
|
||||
if folder.ID == rid {
|
||||
response, err := client.Post("db/override", "")
|
||||
@@ -73,36 +67,10 @@ func (f *folderOverrideCommand) Run(ctx Context) error {
|
||||
if body != "" {
|
||||
errStr += "\nBody: " + body
|
||||
}
|
||||
return errors.New(errStr)
|
||||
return fmt.Errorf(errStr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("Folder %q not found", rid)
|
||||
}
|
||||
|
||||
func (d *defaultIgnoresCommand) Run(ctx Context) error {
|
||||
client, err := ctx.clientFactory.getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dir, file := filepath.Split(d.Path)
|
||||
filesystem := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
|
||||
|
||||
fd, err := filesystem.Open(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
scanner := bufio.NewScanner(fd)
|
||||
var lines []string
|
||||
for scanner.Scan() {
|
||||
lines = append(lines, scanner.Text())
|
||||
}
|
||||
fd.Close()
|
||||
if err := scanner.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = client.PutJSON("config/defaults/ignores", config.Ignores{Lines: lines})
|
||||
return err
|
||||
return fmt.Errorf("Folder " + rid + " not found")
|
||||
}
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
// Copyright (C) 2021 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package cli
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
|
||||
"github.com/alecthomas/kong"
|
||||
)
|
||||
|
||||
type pendingCommand struct {
|
||||
Devices struct{} `cmd:"" help:"Show pending devices"`
|
||||
Folders struct {
|
||||
Device string `help:"Show pending folders offered by given device"`
|
||||
} `cmd:"" help:"Show pending folders"`
|
||||
}
|
||||
|
||||
func (p *pendingCommand) Run(ctx Context, kongCtx *kong.Context) error {
|
||||
indexDumpOutput := indexDumpOutputWrapper(ctx.clientFactory)
|
||||
|
||||
switch kongCtx.Selected().Name {
|
||||
case "devices":
|
||||
return indexDumpOutput("cluster/pending/devices")
|
||||
case "folders":
|
||||
if p.Folders.Device != "" {
|
||||
query := make(url.Values)
|
||||
query.Set("device", p.Folders.Device)
|
||||
return indexDumpOutput("cluster/pending/folders?" + query.Encode())
|
||||
}
|
||||
return indexDumpOutput("cluster/pending/folders")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -7,36 +7,38 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/alecthomas/kong"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
type showCommand struct {
|
||||
Version struct{} `cmd:"" help:"Show syncthing client version"`
|
||||
ConfigStatus struct{} `cmd:"" help:"Show configuration status, whether or not a restart is required for changes to take effect"`
|
||||
System struct{} `cmd:"" help:"Show system status"`
|
||||
Connections struct{} `cmd:"" help:"Report about connections to other devices"`
|
||||
Discovery struct{} `cmd:"" help:"Show the discovered addresses of remote devices (from cache of the running syncthing instance)"`
|
||||
Usage struct{} `cmd:"" help:"Show usage report"`
|
||||
Pending pendingCommand `cmd:"" help:"Pending subcommand group"`
|
||||
}
|
||||
|
||||
func (*showCommand) Run(ctx Context, kongCtx *kong.Context) error {
|
||||
indexDumpOutput := indexDumpOutputWrapper(ctx.clientFactory)
|
||||
|
||||
switch kongCtx.Selected().Name {
|
||||
case "version":
|
||||
return indexDumpOutput("system/version")
|
||||
case "config-status":
|
||||
return indexDumpOutput("config/restart-required")
|
||||
case "system":
|
||||
return indexDumpOutput("system/status")
|
||||
case "connections":
|
||||
return indexDumpOutput("system/connections")
|
||||
case "discovery":
|
||||
return indexDumpOutput("system/discovery")
|
||||
case "usage":
|
||||
return indexDumpOutput("svc/report")
|
||||
}
|
||||
|
||||
return nil
|
||||
var showCommand = cli.Command{
|
||||
Name: "show",
|
||||
HideHelp: true,
|
||||
Usage: "Show command group",
|
||||
Subcommands: []cli.Command{
|
||||
{
|
||||
Name: "version",
|
||||
Usage: "Show syncthing client version",
|
||||
Action: expects(0, indexDumpOutput("system/version")),
|
||||
},
|
||||
{
|
||||
Name: "config-status",
|
||||
Usage: "Show configuration status, whether or not a restart is required for changes to take effect",
|
||||
Action: expects(0, indexDumpOutput("config/restart-required")),
|
||||
},
|
||||
{
|
||||
Name: "system",
|
||||
Usage: "Show system status",
|
||||
Action: expects(0, indexDumpOutput("system/status")),
|
||||
},
|
||||
{
|
||||
Name: "connections",
|
||||
Usage: "Report about connections to other devices",
|
||||
Action: expects(0, indexDumpOutput("system/connections")),
|
||||
},
|
||||
{
|
||||
Name: "usage",
|
||||
Usage: "Show usage report",
|
||||
Action: expects(0, indexDumpOutput("svc/report")),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"mime"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -19,82 +19,79 @@ import (
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
"github.com/syncthing/syncthing/lib/db/backend"
|
||||
"github.com/syncthing/syncthing/lib/locations"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
func responseToBArray(response *http.Response) ([]byte, error) {
|
||||
bytes, err := io.ReadAll(response.Body)
|
||||
bytes, err := ioutil.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes, response.Body.Close()
|
||||
}
|
||||
|
||||
func emptyPost(url string, apiClientFactory *apiClientFactory) error {
|
||||
client, err := apiClientFactory.getClient()
|
||||
if err != nil {
|
||||
func emptyPost(url string) cli.ActionFunc {
|
||||
return func(c *cli.Context) error {
|
||||
client, err := getClientFactory(c).getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = client.Post(url, "")
|
||||
return err
|
||||
}
|
||||
_, err = client.Post(url, "")
|
||||
return err
|
||||
}
|
||||
|
||||
func indexDumpOutputWrapper(apiClientFactory *apiClientFactory) func(url string) error {
|
||||
return func(url string) error {
|
||||
return indexDumpOutput(url, apiClientFactory)
|
||||
}
|
||||
}
|
||||
|
||||
func indexDumpOutput(url string, apiClientFactory *apiClientFactory) error {
|
||||
client, err := apiClientFactory.getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
func indexDumpOutput(url string) cli.ActionFunc {
|
||||
return func(c *cli.Context) error {
|
||||
client, err := getClientFactory(c).getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := client.Get(url)
|
||||
if errors.Is(err, errNotFound) {
|
||||
return errors.New("not found (folder/file not in database)")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return prettyPrintResponse(c, response)
|
||||
}
|
||||
response, err := client.Get(url)
|
||||
if errors.Is(err, errNotFound) {
|
||||
return errors.New("not found (folder/file not in database)")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return prettyPrintResponse(response)
|
||||
}
|
||||
|
||||
func saveToFile(url string, apiClientFactory *apiClientFactory) error {
|
||||
client, err := apiClientFactory.getClient()
|
||||
if err != nil {
|
||||
func saveToFile(url string) cli.ActionFunc {
|
||||
return func(c *cli.Context) error {
|
||||
client, err := getClientFactory(c).getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := client.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, params, err := mime.ParseMediaType(response.Header.Get("Content-Disposition"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
filename := params["filename"]
|
||||
if filename == "" {
|
||||
return errors.New("Missing filename in response")
|
||||
}
|
||||
bs, err := responseToBArray(response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
_, err = f.Write(bs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("Wrote results to", filename)
|
||||
return err
|
||||
}
|
||||
response, err := client.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, params, err := mime.ParseMediaType(response.Header.Get("Content-Disposition"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
filename := params["filename"]
|
||||
if filename == "" {
|
||||
return errors.New("Missing filename in response")
|
||||
}
|
||||
bs, err := responseToBArray(response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = f.Write(bs)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("Wrote results to", filename)
|
||||
return nil
|
||||
}
|
||||
|
||||
func getConfig(c APIClient) (config.Configuration, error) {
|
||||
@@ -108,19 +105,32 @@ func getConfig(c APIClient) (config.Configuration, error) {
|
||||
return cfg, err
|
||||
}
|
||||
err = json.Unmarshal(bytes, &cfg)
|
||||
if err != nil {
|
||||
return config.Configuration{}, err
|
||||
if err == nil {
|
||||
return cfg, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func expects(n int, actionFunc cli.ActionFunc) cli.ActionFunc {
|
||||
return func(ctx *cli.Context) error {
|
||||
if ctx.NArg() != n {
|
||||
plural := ""
|
||||
if n != 1 {
|
||||
plural = "s"
|
||||
}
|
||||
return fmt.Errorf("expected %d argument%s, got %d", n, plural, ctx.NArg())
|
||||
}
|
||||
return actionFunc(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func prettyPrintJSON(data interface{}) error {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(data)
|
||||
}
|
||||
|
||||
func prettyPrintResponse(response *http.Response) error {
|
||||
func prettyPrintResponse(c *cli.Context, response *http.Response) error {
|
||||
bytes, err := responseToBArray(response)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -149,3 +159,7 @@ func nulString(bs []byte) string {
|
||||
func normalizePath(path string) string {
|
||||
return filepath.ToSlash(filepath.Clean(path))
|
||||
}
|
||||
|
||||
func getClientFactory(c *cli.Context) *apiClientFactory {
|
||||
return c.App.Metadata["clientFactory"].(*apiClientFactory)
|
||||
}
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
// Copyright (C) 2021 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package cmdutil
|
||||
|
||||
// CommonOptions are reused among several subcommands
|
||||
type CommonOptions struct {
|
||||
buildCommonOptions
|
||||
ConfDir string `name:"config" placeholder:"PATH" env:"STCONFDIR" help:"Set configuration directory (config and keys)"`
|
||||
HomeDir string `name:"home" placeholder:"PATH" env:"STHOMEDIR" help:"Set configuration and data directory"`
|
||||
NoDefaultFolder bool `env:"STNODEFAULTFOLDER" help:"Don't create the \"default\" folder on first startup"`
|
||||
SkipPortProbing bool `help:"Don't try to find free ports for GUI and listen addresses on first startup"`
|
||||
}
|
||||
@@ -18,9 +18,9 @@ func SetConfigDataLocationsFromFlags(homeDir, confDir, dataDir string) error {
|
||||
dataSet := dataDir != ""
|
||||
switch {
|
||||
case dataSet != confSet:
|
||||
return errors.New("either both or none of --config and --data must be given, use --home to set both at once")
|
||||
return errors.New("either both or none of -conf and -data must be given, use -home to set both at once")
|
||||
case homeSet && dataSet:
|
||||
return errors.New("--home must not be used together with --config and --data")
|
||||
return errors.New("-home must not be used together with -conf and -data")
|
||||
case homeSet:
|
||||
confDir = homeDir
|
||||
dataDir = homeDir
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -61,7 +62,7 @@ func uploadPanicLogs(ctx context.Context, urlBase, dir string) {
|
||||
// the log contents. A HEAD request is made to see if the log has already
|
||||
// been reported. If not, a PUT is made with the log contents.
|
||||
func uploadPanicLog(ctx context.Context, urlBase, file string) error {
|
||||
data, err := os.ReadFile(file)
|
||||
data, err := ioutil.ReadFile(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -10,11 +10,10 @@ package decrypt
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
@@ -35,7 +34,6 @@ type CLI struct {
|
||||
TokenPath string `placeholder:"PATH" help:"Path to the token file within the folder (used to determine folder ID)"`
|
||||
|
||||
folderKey *[32]byte
|
||||
keyGen *protocol.KeyGenerator
|
||||
}
|
||||
|
||||
type storedEncryptionToken struct {
|
||||
@@ -47,7 +45,7 @@ func (c *CLI) Run() error {
|
||||
log.SetFlags(0)
|
||||
|
||||
if c.To == "" && !c.VerifyOnly {
|
||||
return errors.New("must set --to or --verify-only")
|
||||
return fmt.Errorf("must set --to or --verify")
|
||||
}
|
||||
|
||||
if c.TokenPath == "" {
|
||||
@@ -69,8 +67,7 @@ func (c *CLI) Run() error {
|
||||
}
|
||||
}
|
||||
|
||||
c.keyGen = protocol.NewKeyGenerator()
|
||||
c.folderKey = c.keyGen.KeyFromPassword(c.FolderID, c.Password)
|
||||
c.folderKey = protocol.KeyFromPassword(c.FolderID, c.Password)
|
||||
|
||||
return c.walk()
|
||||
}
|
||||
@@ -115,7 +112,7 @@ func (c *CLI) withContinue(err error) error {
|
||||
// error.
|
||||
func (c *CLI) getFolderID() (string, error) {
|
||||
tokenPath := filepath.Join(c.Path, c.TokenPath)
|
||||
bs, err := os.ReadFile(tokenPath)
|
||||
bs, err := ioutil.ReadFile(tokenPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("reading folder token: %w", err)
|
||||
}
|
||||
@@ -131,9 +128,6 @@ func (c *CLI) getFolderID() (string, error) {
|
||||
// process handles the file named path in srcFs, decrypting it into dstFs
|
||||
// unless dstFs is nil.
|
||||
func (c *CLI) process(srcFs fs.Filesystem, dstFs fs.Filesystem, path string) error {
|
||||
// Which filemode bits to preserve
|
||||
const retainBits = fs.ModePerm | fs.ModeSetgid | fs.ModeSetuid | fs.ModeSticky
|
||||
|
||||
if c.Verbose {
|
||||
log.Printf("Processing %q", path)
|
||||
}
|
||||
@@ -144,7 +138,7 @@ func (c *CLI) process(srcFs fs.Filesystem, dstFs fs.Filesystem, path string) err
|
||||
}
|
||||
defer encFd.Close()
|
||||
|
||||
encFi, err := loadEncryptedFileInfo(encFd)
|
||||
encFi, err := c.loadEncryptedFileInfo(encFd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: loading metadata trailer: %w", path, err)
|
||||
}
|
||||
@@ -153,7 +147,7 @@ func (c *CLI) process(srcFs fs.Filesystem, dstFs fs.Filesystem, path string) err
|
||||
// in native format, while protocol expects wire format (slashes).
|
||||
encFi.Name = osutil.NormalizedFilename(encFi.Name)
|
||||
|
||||
plainFi, err := protocol.DecryptFileInfo(c.keyGen, *encFi, c.folderKey)
|
||||
plainFi, err := protocol.DecryptFileInfo(*encFi, c.folderKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: decrypting metadata: %w", path, err)
|
||||
}
|
||||
@@ -164,7 +158,7 @@ func (c *CLI) process(srcFs fs.Filesystem, dstFs fs.Filesystem, path string) err
|
||||
|
||||
var plainFd fs.File
|
||||
if dstFs != nil {
|
||||
if err := dstFs.MkdirAll(filepath.Dir(plainFi.Name), 0o700); err != nil {
|
||||
if err := dstFs.MkdirAll(filepath.Dir(plainFi.Name), 0700); err != nil {
|
||||
return fmt.Errorf("%s: %w", plainFi.Name, err)
|
||||
}
|
||||
|
||||
@@ -173,9 +167,6 @@ func (c *CLI) process(srcFs fs.Filesystem, dstFs fs.Filesystem, path string) err
|
||||
return fmt.Errorf("%s: %w", plainFi.Name, err)
|
||||
}
|
||||
defer plainFd.Close() // also closed explicitly in the return
|
||||
if err := dstFs.Chmod(plainFi.Name, fs.FileMode(plainFi.Permissions&uint32(retainBits))); err != nil {
|
||||
return fmt.Errorf("%s: %w", plainFi.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := c.decryptFile(encFi, &plainFi, encFd, plainFd); err != nil {
|
||||
@@ -192,12 +183,7 @@ func (c *CLI) process(srcFs fs.Filesystem, dstFs fs.Filesystem, path string) err
|
||||
}
|
||||
|
||||
if plainFd != nil {
|
||||
if err := plainFd.Close(); err != nil {
|
||||
return fmt.Errorf("%s: %w", plainFi.Name, err)
|
||||
}
|
||||
if err := dstFs.Chtimes(plainFi.Name, plainFi.ModTime(), plainFi.ModTime()); err != nil {
|
||||
return fmt.Errorf("%s: %w", plainFi.Name, err)
|
||||
}
|
||||
return plainFd.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -211,7 +197,7 @@ func (c *CLI) decryptFile(encFi *protocol.FileInfo, plainFi *protocol.FileInfo,
|
||||
return fmt.Errorf("block count mismatch: encrypted %d != plaintext %d", len(encFi.Blocks), len(plainFi.Blocks))
|
||||
}
|
||||
|
||||
fileKey := c.keyGen.FileKey(plainFi.Name, c.folderKey)
|
||||
fileKey := protocol.FileKey(plainFi.Name, c.folderKey)
|
||||
for i, encBlock := range encFi.Blocks {
|
||||
// Read the encrypted block
|
||||
buf := make([]byte, encBlock.Size)
|
||||
@@ -260,7 +246,7 @@ func (c *CLI) decryptFile(encFi *protocol.FileInfo, plainFi *protocol.FileInfo,
|
||||
|
||||
// loadEncryptedFileInfo loads the encrypted FileInfo trailer from a file on
|
||||
// disk.
|
||||
func loadEncryptedFileInfo(fd fs.File) (*protocol.FileInfo, error) {
|
||||
func (c *CLI) loadEncryptedFileInfo(fd fs.File) (*protocol.FileInfo, error) {
|
||||
// Seek to the size of the trailer block
|
||||
if _, err := fd.Seek(-4, io.SeekEnd); err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
// Copyright (C) 2021 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
// Package generate implements the `syncthing generate` subcommand.
|
||||
package generate
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
"github.com/syncthing/syncthing/lib/events"
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
"github.com/syncthing/syncthing/lib/locations"
|
||||
"github.com/syncthing/syncthing/lib/logger"
|
||||
"github.com/syncthing/syncthing/lib/osutil"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/syncthing"
|
||||
)
|
||||
|
||||
type CLI struct {
|
||||
cmdutil.CommonOptions
|
||||
GUIUser string `placeholder:"STRING" help:"Specify new GUI authentication user name"`
|
||||
GUIPassword string `placeholder:"STRING" help:"Specify new GUI authentication password (use - to read from standard input)"`
|
||||
}
|
||||
|
||||
func (c *CLI) Run(l logger.Logger) error {
|
||||
if c.HideConsole {
|
||||
osutil.HideConsole()
|
||||
}
|
||||
|
||||
if c.HomeDir != "" {
|
||||
if c.ConfDir != "" {
|
||||
return errors.New("--home must not be used together with --config")
|
||||
}
|
||||
c.ConfDir = c.HomeDir
|
||||
}
|
||||
if c.ConfDir == "" {
|
||||
c.ConfDir = locations.GetBaseDir(locations.ConfigBaseDir)
|
||||
}
|
||||
|
||||
// Support reading the password from a pipe or similar
|
||||
if c.GUIPassword == "-" {
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
password, _, err := reader.ReadLine()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed reading GUI password: %w", err)
|
||||
}
|
||||
c.GUIPassword = string(password)
|
||||
}
|
||||
|
||||
if err := Generate(l, c.ConfDir, c.GUIUser, c.GUIPassword, c.NoDefaultFolder, c.SkipPortProbing); err != nil {
|
||||
return fmt.Errorf("failed to generate config and keys: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func Generate(l logger.Logger, confDir, guiUser, guiPassword string, noDefaultFolder, skipPortProbing bool) error {
|
||||
dir, err := fs.ExpandTilde(confDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := syncthing.EnsureDir(dir, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
locations.SetBaseDir(locations.ConfigBaseDir, dir)
|
||||
|
||||
var myID protocol.DeviceID
|
||||
certFile, keyFile := locations.Get(locations.CertFile), locations.Get(locations.KeyFile)
|
||||
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
|
||||
if err == nil {
|
||||
l.Warnln("Key exists; will not overwrite.")
|
||||
} else {
|
||||
cert, err = syncthing.GenerateCertificate(certFile, keyFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create certificate: %w", err)
|
||||
}
|
||||
}
|
||||
myID = protocol.NewDeviceID(cert.Certificate[0])
|
||||
l.Infoln("Device ID:", myID)
|
||||
|
||||
cfgFile := locations.Get(locations.ConfigFile)
|
||||
cfg, _, err := config.Load(cfgFile, myID, events.NoopLogger)
|
||||
if fs.IsNotExist(err) {
|
||||
if cfg, err = syncthing.DefaultConfig(cfgFile, myID, events.NoopLogger, noDefaultFolder, skipPortProbing); err != nil {
|
||||
return fmt.Errorf("create config: %w", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go cfg.Serve(ctx)
|
||||
defer cancel()
|
||||
|
||||
var updateErr error
|
||||
waiter, err := cfg.Modify(func(cfg *config.Configuration) {
|
||||
updateErr = updateGUIAuthentication(l, &cfg.GUI, guiUser, guiPassword)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("modify config: %w", err)
|
||||
}
|
||||
|
||||
waiter.Wait()
|
||||
if updateErr != nil {
|
||||
return updateErr
|
||||
}
|
||||
if err := cfg.Save(); err != nil {
|
||||
return fmt.Errorf("save config: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func updateGUIAuthentication(l logger.Logger, guiCfg *config.GUIConfiguration, guiUser, guiPassword string) error {
|
||||
if guiUser != "" && guiCfg.User != guiUser {
|
||||
guiCfg.User = guiUser
|
||||
l.Infoln("Updated GUI authentication user name:", guiUser)
|
||||
}
|
||||
|
||||
if guiPassword != "" && guiCfg.Password != guiPassword {
|
||||
if err := guiCfg.SetPassword(guiPassword); err != nil {
|
||||
return fmt.Errorf("failed to set GUI authentication password: %w", err)
|
||||
}
|
||||
l.Infoln("Updated GUI authentication password.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -10,9 +10,9 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
_ "net/http/pprof" // Need to import this to support STPROFILER.
|
||||
@@ -36,7 +36,6 @@ import (
|
||||
"github.com/syncthing/syncthing/cmd/syncthing/cli"
|
||||
"github.com/syncthing/syncthing/cmd/syncthing/cmdutil"
|
||||
"github.com/syncthing/syncthing/cmd/syncthing/decrypt"
|
||||
"github.com/syncthing/syncthing/cmd/syncthing/generate"
|
||||
"github.com/syncthing/syncthing/lib/build"
|
||||
"github.com/syncthing/syncthing/lib/config"
|
||||
"github.com/syncthing/syncthing/lib/db"
|
||||
@@ -50,11 +49,16 @@ import (
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/svcutil"
|
||||
"github.com/syncthing/syncthing/lib/syncthing"
|
||||
"github.com/syncthing/syncthing/lib/tlsutil"
|
||||
"github.com/syncthing/syncthing/lib/upgrade"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
sigTerm = syscall.Signal(15)
|
||||
tlsDefaultCommonName = "syncthing"
|
||||
deviceCertLifetimeDays = 20 * 365
|
||||
sigTerm = syscall.Signal(15)
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -67,14 +71,14 @@ The --logflags value is a sum of the following:
|
||||
8 Long filename
|
||||
16 Short filename
|
||||
|
||||
I.e. to prefix each log line with time and filename, set --logflags=18 (2 + 16
|
||||
from above). The value 0 is used to disable all of the above. The default is
|
||||
to show date and time (3).
|
||||
I.e. to prefix each log line with date and time, set --logflags=3 (1 + 2 from
|
||||
above). The value 0 is used to disable all of the above. The default is to
|
||||
show time only (2).
|
||||
|
||||
Logging always happens to the command line (stdout) and optionally to the
|
||||
file at the path specified by --logfile=path. In addition to an path, the special
|
||||
file at the path specified by -logfile=path. In addition to an path, the special
|
||||
values "default" and "-" may be used. The former logs to DATADIR/syncthing.log
|
||||
(see --data), which is the default on Windows, and the latter only to stdout,
|
||||
(see -data-dir), which is the default on Windows, and the latter only to stdout,
|
||||
no file, which is the default anywhere else.
|
||||
|
||||
|
||||
@@ -88,6 +92,9 @@ above.
|
||||
STTRACE A comma separated string of facilities to trace. The valid
|
||||
facility strings are listed below.
|
||||
|
||||
STDEADLOCKTIMEOUT Used for debugging internal deadlocks; sets debug
|
||||
sensitivity. Use only under direction of a developer.
|
||||
|
||||
STLOCKTHRESHOLD Used for debugging internal deadlocks; sets debug
|
||||
sensitivity. Use only under direction of a developer.
|
||||
|
||||
@@ -96,11 +103,6 @@ above.
|
||||
"minio" for the github.com/minio/sha256-simd implementation,
|
||||
and blank (the default) for auto detection.
|
||||
|
||||
STVERSIONEXTRA Add extra information to the version string in logs and the
|
||||
version line in the GUI. Can be set to the name of a wrapper
|
||||
or tool controlling syncthing to communicate this to the end
|
||||
user.
|
||||
|
||||
GOMAXPROCS Set the maximum number of CPU cores to use. Defaults to all
|
||||
available CPU cores.
|
||||
|
||||
@@ -133,30 +135,32 @@ var (
|
||||
// commands and options here are top level commands to syncthing.
|
||||
// Cli is just a placeholder for the help text (see main).
|
||||
var entrypoint struct {
|
||||
Serve serveOptions `cmd:"" help:"Run Syncthing"`
|
||||
Generate generate.CLI `cmd:"" help:"Generate key and config, then exit"`
|
||||
Decrypt decrypt.CLI `cmd:"" help:"Decrypt or verify an encrypted folder"`
|
||||
Cli cli.CLI `cmd:"" help:"Command line interface for Syncthing"`
|
||||
Serve serveOptions `cmd:"" help:"Run Syncthing"`
|
||||
Decrypt decrypt.CLI `cmd:"" help:"Decrypt or verify an encrypted folder"`
|
||||
Cli struct{} `cmd:"" help:"Command line interface for Syncthing"`
|
||||
}
|
||||
|
||||
// serveOptions are the options for the `syncthing serve` command.
|
||||
type serveOptions struct {
|
||||
cmdutil.CommonOptions
|
||||
buildServeOptions
|
||||
AllowNewerConfig bool `help:"Allow loading newer than current config version"`
|
||||
Audit bool `help:"Write events to audit file"`
|
||||
AuditFile string `name:"auditfile" placeholder:"PATH" help:"Specify audit file (use \"-\" for stdout, \"--\" for stderr)"`
|
||||
BrowserOnly bool `help:"Open GUI in browser"`
|
||||
DataDir string `name:"data" placeholder:"PATH" env:"STDATADIR" help:"Set data directory (database and logs)"`
|
||||
ConfDir string `name:"config" placeholder:"PATH" help:"Set configuration directory (config and keys)"`
|
||||
DataDir string `name:"data" placeholder:"PATH" help:"Set data directory (database and logs)"`
|
||||
DeviceID bool `help:"Show the device ID"`
|
||||
GenerateDir string `name:"generate" placeholder:"PATH" help:"Generate key and config in specified dir, then exit"` // DEPRECATED: replaced by subcommand!
|
||||
GenerateDir string `name:"generate" placeholder:"PATH" help:"Generate key and config in specified dir, then exit"`
|
||||
GUIAddress string `name:"gui-address" placeholder:"URL" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")"`
|
||||
GUIAPIKey string `name:"gui-apikey" placeholder:"API-KEY" help:"Override GUI API key"`
|
||||
HomeDir string `name:"home" placeholder:"PATH" help:"Set configuration and data directory"`
|
||||
LogFile string `name:"logfile" default:"${logFile}" placeholder:"PATH" help:"Log file name (see below)"`
|
||||
LogFlags int `name:"logflags" default:"${logFlags}" placeholder:"BITS" help:"Select information in log line prefix (see below)"`
|
||||
LogMaxFiles int `placeholder:"N" default:"${logMaxFiles}" name:"log-max-old-files" help:"Number of old files to keep (zero to keep only current)"`
|
||||
LogMaxSize int `placeholder:"BYTES" default:"${logMaxSize}" help:"Maximum size of any file (zero to disable log rotation)"`
|
||||
NoBrowser bool `help:"Do not start browser"`
|
||||
NoRestart bool `env:"STNORESTART" help:"Do not restart Syncthing when exiting due to API/GUI command, upgrade, or crash"`
|
||||
NoDefaultFolder bool `env:"STNODEFAULTFOLDER" help:"Don't create the \"default\" folder on first startup"`
|
||||
NoUpgrade bool `env:"STNOUPGRADE" help:"Disable automatic upgrades"`
|
||||
Paths bool `help:"Show configuration paths"`
|
||||
Paused bool `help:"Start with all devices and folders paused"`
|
||||
@@ -170,10 +174,11 @@ type serveOptions struct {
|
||||
// Debug options below
|
||||
DebugDBIndirectGCInterval time.Duration `env:"STGCINDIRECTEVERY" help:"Database indirection GC interval"`
|
||||
DebugDBRecheckInterval time.Duration `env:"STRECHECKDBEVERY" help:"Database metadata recalculation interval"`
|
||||
DebugDeadlockTimeout int `placeholder:"SECONDS" env:"STDEADLOCKTIMEOUT" help:"Used for debugging internal deadlocks"`
|
||||
DebugGUIAssetsDir string `placeholder:"PATH" help:"Directory to load GUI assets from" env:"STGUIASSETS"`
|
||||
DebugPerfStats bool `env:"STPERFSTATS" help:"Write running performance statistics to perf-$pid.csv (Unix only)"`
|
||||
DebugProfileBlock bool `env:"STBLOCKPROFILE" help:"Write block profiles to block-$pid-$timestamp.pprof every 20 seconds"`
|
||||
DebugProfileCPU bool `help:"Write a CPU profile to cpu-$pid.pprof on exit" env:"STCPUPROFILE"`
|
||||
DebugProfileCPU bool `help:"Write a CPU profile to cpu-$pid.pprof on exit" env:"CPUPROFILE"`
|
||||
DebugProfileHeap bool `env:"STHEAPPROFILE" help:"Write heap profiles to heap-$pid-$timestamp.pprof each time heap usage increases"`
|
||||
DebugProfilerListen string `placeholder:"ADDR" env:"STPROFILER" help:"Network profiler listen address"`
|
||||
DebugResetDatabase bool `name:"reset-database" help:"Reset the database, forcing a full rescan and resync"`
|
||||
@@ -187,7 +192,7 @@ type serveOptions struct {
|
||||
func defaultVars() kong.Vars {
|
||||
vars := kong.Vars{}
|
||||
|
||||
vars["logFlags"] = strconv.Itoa(logger.DefaultFlags)
|
||||
vars["logFlags"] = strconv.Itoa(log.Ltime)
|
||||
vars["logMaxSize"] = strconv.Itoa(10 << 20) // 10 MiB
|
||||
vars["logMaxFiles"] = "3" // plus the current one
|
||||
|
||||
@@ -199,7 +204,7 @@ func defaultVars() kong.Vars {
|
||||
// Windows, the "default" options.logFile will later be replaced with the
|
||||
// default path, unless the user has manually specified "-" or
|
||||
// something else.
|
||||
if build.IsWindows {
|
||||
if runtime.GOOS == "windows" {
|
||||
vars["logFile"] = "default"
|
||||
} else {
|
||||
vars["logFile"] = "-"
|
||||
@@ -209,6 +214,17 @@ func defaultVars() kong.Vars {
|
||||
}
|
||||
|
||||
func main() {
|
||||
// The "cli" subcommand uses a different command line parser, and e.g. help
|
||||
// gets mangled when integrating it as a subcommand -> detect it here at the
|
||||
// beginning.
|
||||
if len(os.Args) > 1 && os.Args[1] == "cli" {
|
||||
if err := cli.Run(); err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// First some massaging of the raw command line to fit the new model.
|
||||
// Basically this means adding the default command at the front, and
|
||||
// converting -options to --options.
|
||||
@@ -234,22 +250,13 @@ func main() {
|
||||
|
||||
// Create a parser with an overridden help function to print our extra
|
||||
// help info.
|
||||
parser, err := kong.New(
|
||||
&entrypoint,
|
||||
kong.ConfigureHelp(kong.HelpOptions{
|
||||
NoExpandSubcommands: true,
|
||||
Compact: true,
|
||||
}),
|
||||
kong.Help(helpHandler),
|
||||
defaultVars(),
|
||||
)
|
||||
parser, err := kong.New(&entrypoint, kong.Help(helpHandler), defaultVars())
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
ctx, err := parser.Parse(args)
|
||||
parser.FatalIfErrorf(err)
|
||||
ctx.BindTo(l, (*logger.Logger)(nil)) // main logger available to subcommands
|
||||
err = ctx.Run()
|
||||
parser.FatalIfErrorf(err)
|
||||
}
|
||||
@@ -290,25 +297,16 @@ func (options serveOptions) Run() error {
|
||||
os.Exit(svcutil.ExitError.AsInt())
|
||||
}
|
||||
|
||||
// Treat an explicitly empty log file name as no log file
|
||||
if options.LogFile == "" {
|
||||
options.LogFile = "-"
|
||||
}
|
||||
if options.LogFile != "default" {
|
||||
if options.LogFile == "default" || options.LogFile == "" {
|
||||
// We must set this *after* expandLocations above.
|
||||
if err := locations.Set(locations.LogFile, options.LogFile); err != nil {
|
||||
l.Warnln("Setting log file path:", err)
|
||||
os.Exit(svcutil.ExitError.AsInt())
|
||||
}
|
||||
// Handling an empty value is for backwards compatibility (<1.4.1).
|
||||
options.LogFile = locations.Get(locations.LogFile)
|
||||
}
|
||||
|
||||
if options.DebugGUIAssetsDir != "" {
|
||||
if options.DebugGUIAssetsDir == "" {
|
||||
// The asset dir is blank if STGUIASSETS wasn't set, in which case we
|
||||
// should look for extra assets in the default place.
|
||||
if err := locations.Set(locations.GUIAssets, options.DebugGUIAssetsDir); err != nil {
|
||||
l.Warnln("Setting GUI assets path:", err)
|
||||
os.Exit(svcutil.ExitError.AsInt())
|
||||
}
|
||||
options.DebugGUIAssetsDir = locations.Get(locations.GUIAssets)
|
||||
}
|
||||
|
||||
if options.Version {
|
||||
@@ -317,7 +315,7 @@ func (options serveOptions) Run() error {
|
||||
}
|
||||
|
||||
if options.Paths {
|
||||
fmt.Print(locations.PrettyPaths())
|
||||
showPaths(options)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -336,7 +334,7 @@ func (options serveOptions) Run() error {
|
||||
}
|
||||
|
||||
if options.BrowserOnly {
|
||||
if err := openGUI(); err != nil {
|
||||
if err := openGUI(protocol.EmptyDeviceID); err != nil {
|
||||
l.Warnln("Failed to open web UI:", err)
|
||||
os.Exit(svcutil.ExitError.AsInt())
|
||||
}
|
||||
@@ -344,7 +342,7 @@ func (options serveOptions) Run() error {
|
||||
}
|
||||
|
||||
if options.GenerateDir != "" {
|
||||
if err := generate.Generate(l, options.GenerateDir, "", "", options.NoDefaultFolder, options.SkipPortProbing); err != nil {
|
||||
if err := generate(options.GenerateDir, options.NoDefaultFolder); err != nil {
|
||||
l.Warnln("Failed to generate config and keys:", err)
|
||||
os.Exit(svcutil.ExitError.AsInt())
|
||||
}
|
||||
@@ -352,7 +350,7 @@ func (options serveOptions) Run() error {
|
||||
}
|
||||
|
||||
// Ensure that our home directory exists.
|
||||
if err := syncthing.EnsureDir(locations.GetBaseDir(locations.ConfigBaseDir), 0o700); err != nil {
|
||||
if err := ensureDir(locations.GetBaseDir(locations.ConfigBaseDir), 0700); err != nil {
|
||||
l.Warnln("Failure on home directory:", err)
|
||||
os.Exit(svcutil.ExitError.AsInt())
|
||||
}
|
||||
@@ -413,13 +411,13 @@ func (options serveOptions) Run() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func openGUI() error {
|
||||
cfg, err := loadOrDefaultConfig()
|
||||
func openGUI(myID protocol.DeviceID) error {
|
||||
cfg, err := loadOrDefaultConfig(myID, events.NoopLogger, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if guiCfg := cfg.GUI(); guiCfg.Enabled {
|
||||
if err := openURL(guiCfg.URL()); err != nil {
|
||||
if cfg.GUI().Enabled {
|
||||
if err := openURL(cfg.GUI().URL()); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
@@ -428,6 +426,46 @@ func openGUI() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func generate(generateDir string, noDefaultFolder bool) error {
|
||||
dir, err := fs.ExpandTilde(generateDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ensureDir(dir, 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var myID protocol.DeviceID
|
||||
certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem")
|
||||
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
|
||||
if err == nil {
|
||||
l.Warnln("Key exists; will not overwrite.")
|
||||
} else {
|
||||
cert, err = tlsutil.NewCertificate(certFile, keyFile, tlsDefaultCommonName, deviceCertLifetimeDays)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "create certificate")
|
||||
}
|
||||
}
|
||||
myID = protocol.NewDeviceID(cert.Certificate[0])
|
||||
l.Infoln("Device ID:", myID)
|
||||
|
||||
cfgFile := filepath.Join(dir, "config.xml")
|
||||
if _, err := os.Stat(cfgFile); err == nil {
|
||||
l.Warnln("Config exists; will not overwrite.")
|
||||
return nil
|
||||
}
|
||||
cfg, err := syncthing.DefaultConfig(cfgFile, myID, events.NoopLogger, noDefaultFolder)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = cfg.Save()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "save config")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func debugFacilities() string {
|
||||
facilities := l.Facilities()
|
||||
|
||||
@@ -459,7 +497,7 @@ func (e *errNoUpgrade) Error() string {
|
||||
}
|
||||
|
||||
func checkUpgrade() (upgrade.Release, error) {
|
||||
cfg, err := loadOrDefaultConfig()
|
||||
cfg, err := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger, true)
|
||||
if err != nil {
|
||||
return upgrade.Release{}, err
|
||||
}
|
||||
@@ -478,11 +516,7 @@ func checkUpgrade() (upgrade.Release, error) {
|
||||
}
|
||||
|
||||
func upgradeViaRest() error {
|
||||
cfg, err := loadOrDefaultConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg, _ := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger, true)
|
||||
u, err := url.Parse(cfg.GUI().URL())
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -506,7 +540,7 @@ func upgradeViaRest() error {
|
||||
return err
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
bs, err := io.ReadAll(resp.Body)
|
||||
bs, err := ioutil.ReadAll(resp.Body)
|
||||
defer resp.Body.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -558,7 +592,7 @@ func syncthingMain(options serveOptions) {
|
||||
evLogger := events.NewLogger()
|
||||
earlyService.Add(evLogger)
|
||||
|
||||
cfgWrapper, err := syncthing.LoadConfigAtStartup(locations.Get(locations.ConfigFile), cert, evLogger, options.AllowNewerConfig, options.NoDefaultFolder, options.SkipPortProbing)
|
||||
cfgWrapper, err := syncthing.LoadConfigAtStartup(locations.Get(locations.ConfigFile), cert, evLogger, options.AllowNewerConfig, options.NoDefaultFolder)
|
||||
if err != nil {
|
||||
l.Warnln("Failed to initialize config:", err)
|
||||
os.Exit(svcutil.ExitError.AsInt())
|
||||
@@ -590,7 +624,7 @@ func syncthingMain(options serveOptions) {
|
||||
}
|
||||
|
||||
// Check if auto-upgrades is possible, and if yes, and it's enabled do an initial
|
||||
// upgrade immediately. The auto-upgrade routine can only be started
|
||||
// upgrade immedately. The auto-upgrade routine can only be started
|
||||
// later after App is initialised.
|
||||
|
||||
autoUpgradePossible := autoUpgradePossible(options)
|
||||
@@ -619,6 +653,8 @@ func syncthingMain(options serveOptions) {
|
||||
}
|
||||
|
||||
appOpts := syncthing.Options{
|
||||
AssetDir: options.DebugGUIAssetsDir,
|
||||
DeadlockTimeoutS: options.DebugDeadlockTimeout,
|
||||
NoUpgrade: options.NoUpgrade,
|
||||
ProfilerAddr: options.DebugProfilerListen,
|
||||
ResetDeltaIdxs: options.DebugResetDeltaIdxs,
|
||||
@@ -629,6 +665,10 @@ func syncthingMain(options serveOptions) {
|
||||
if options.Audit {
|
||||
appOpts.AuditWriter = auditWriter(options.AuditFile)
|
||||
}
|
||||
if t := os.Getenv("STDEADLOCKTIMEOUT"); t != "" {
|
||||
secs, _ := strconv.Atoi(t)
|
||||
appOpts.DeadlockTimeoutS = secs
|
||||
}
|
||||
if dur, err := time.ParseDuration(os.Getenv("STRECHECKDBEVERY")); err == nil {
|
||||
appOpts.DBRecheckInterval = dur
|
||||
}
|
||||
@@ -648,7 +688,7 @@ func syncthingMain(options serveOptions) {
|
||||
|
||||
setupSignalHandling(app)
|
||||
|
||||
if os.Getenv("GOMAXPROCS") == "" {
|
||||
if len(os.Getenv("GOMAXPROCS")) == 0 {
|
||||
runtime.GOMAXPROCS(runtime.NumCPU())
|
||||
}
|
||||
|
||||
@@ -664,6 +704,8 @@ func syncthingMain(options serveOptions) {
|
||||
}
|
||||
}
|
||||
|
||||
go standbyMonitor(app, cfgWrapper)
|
||||
|
||||
if err := app.Start(); err != nil {
|
||||
os.Exit(svcutil.ExitError.AsInt())
|
||||
}
|
||||
@@ -710,14 +752,12 @@ func setupSignalHandling(app *syncthing.App) {
|
||||
}()
|
||||
}
|
||||
|
||||
// loadOrDefaultConfig creates a temporary, minimal configuration wrapper if no file
|
||||
// exists. As it disregards some command-line options, that should never be persisted.
|
||||
func loadOrDefaultConfig() (config.Wrapper, error) {
|
||||
func loadOrDefaultConfig(myID protocol.DeviceID, evLogger events.Logger, noDefaultFolder bool) (config.Wrapper, error) {
|
||||
cfgFile := locations.Get(locations.ConfigFile)
|
||||
cfg, _, err := config.Load(cfgFile, protocol.EmptyDeviceID, events.NoopLogger)
|
||||
cfg, _, err := config.Load(cfgFile, myID, evLogger)
|
||||
|
||||
if err != nil {
|
||||
newCfg := config.New(protocol.EmptyDeviceID)
|
||||
return config.Wrap(cfgFile, newCfg, protocol.EmptyDeviceID, events.NoopLogger), nil
|
||||
cfg, err = syncthing.DefaultConfig(cfgFile, myID, evLogger, noDefaultFolder)
|
||||
}
|
||||
|
||||
return cfg, err
|
||||
@@ -742,7 +782,7 @@ func auditWriter(auditFile string) io.Writer {
|
||||
} else {
|
||||
auditFlags = os.O_WRONLY | os.O_CREATE | os.O_APPEND
|
||||
}
|
||||
fd, err = os.OpenFile(auditFile, auditFlags, 0o600)
|
||||
fd, err = os.OpenFile(auditFile, auditFlags, 0600)
|
||||
if err != nil {
|
||||
l.Warnln("Audit:", err)
|
||||
os.Exit(svcutil.ExitError.AsInt())
|
||||
@@ -759,6 +799,49 @@ func resetDB() error {
|
||||
return os.RemoveAll(locations.Get(locations.Database))
|
||||
}
|
||||
|
||||
func ensureDir(dir string, mode fs.FileMode) error {
|
||||
fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir)
|
||||
err := fs.MkdirAll(".", mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if fi, err := fs.Stat("."); err == nil {
|
||||
// Apprently the stat may fail even though the mkdirall passed. If it
|
||||
// does, we'll just assume things are in order and let other things
|
||||
// fail (like loading or creating the config...).
|
||||
currentMode := fi.Mode() & 0777
|
||||
if currentMode != mode {
|
||||
err := fs.Chmod(".", mode)
|
||||
// This can fail on crappy filesystems, nothing we can do about it.
|
||||
if err != nil {
|
||||
l.Warnln(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func standbyMonitor(app *syncthing.App, cfg config.Wrapper) {
|
||||
restartDelay := 60 * time.Second
|
||||
now := time.Now()
|
||||
for {
|
||||
time.Sleep(10 * time.Second)
|
||||
if time.Since(now) > 2*time.Minute && cfg.Options().RestartOnWakeup {
|
||||
l.Infof("Paused state detected, possibly woke up from standby. Restarting in %v.", restartDelay)
|
||||
|
||||
// We most likely just woke from standby. If we restart
|
||||
// immediately chances are we won't have networking ready. Give
|
||||
// things a moment to stabilize.
|
||||
time.Sleep(restartDelay)
|
||||
|
||||
app.Stop(svcutil.ExitRestart)
|
||||
return
|
||||
}
|
||||
now = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
func autoUpgradePossible(options serveOptions) bool {
|
||||
if upgrade.DisabledByCompilation {
|
||||
return false
|
||||
@@ -890,6 +973,16 @@ func cleanConfigDirectory() {
|
||||
}
|
||||
}
|
||||
|
||||
func showPaths(options serveOptions) {
|
||||
fmt.Printf("Configuration file:\n\t%s\n\n", locations.Get(locations.ConfigFile))
|
||||
fmt.Printf("Database directory:\n\t%s\n\n", locations.Get(locations.Database))
|
||||
fmt.Printf("Device private key & certificate files:\n\t%s\n\t%s\n\n", locations.Get(locations.KeyFile), locations.Get(locations.CertFile))
|
||||
fmt.Printf("HTTPS private key & certificate files:\n\t%s\n\t%s\n\n", locations.Get(locations.HTTPSKeyFile), locations.Get(locations.HTTPSCertFile))
|
||||
fmt.Printf("Log file:\n\t%s\n\n", options.LogFile)
|
||||
fmt.Printf("GUI override directory:\n\t%s\n\n", options.DebugGUIAssetsDir)
|
||||
fmt.Printf("Default sync folder directory:\n\t%s\n\n", locations.Get(locations.DefFolder))
|
||||
}
|
||||
|
||||
func setPauseState(cfgWrapper config.Wrapper, paused bool) {
|
||||
_, err := cfgWrapper.Modify(func(cfg *config.Configuration) {
|
||||
for i := range cfg.Devices {
|
||||
|
||||
@@ -15,14 +15,16 @@ import (
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/build"
|
||||
"github.com/syncthing/syncthing/lib/events"
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
"github.com/syncthing/syncthing/lib/locations"
|
||||
"github.com/syncthing/syncthing/lib/osutil"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/svcutil"
|
||||
"github.com/syncthing/syncthing/lib/sync"
|
||||
)
|
||||
@@ -48,7 +50,7 @@ func monitorMain(options serveOptions) {
|
||||
|
||||
var dst io.Writer = os.Stdout
|
||||
|
||||
logFile := locations.Get(locations.LogFile)
|
||||
logFile := options.LogFile
|
||||
if logFile != "-" {
|
||||
if expanded, err := fs.ExpandTilde(logFile); err == nil {
|
||||
logFile = expanded
|
||||
@@ -66,7 +68,7 @@ func monitorMain(options serveOptions) {
|
||||
if err != nil {
|
||||
l.Warnln("Failed to setup logging to file, proceeding with logging to stdout only:", err)
|
||||
} else {
|
||||
if build.IsWindows {
|
||||
if runtime.GOOS == "windows" {
|
||||
// Translate line breaks to Windows standard
|
||||
fileDst = osutil.ReplacingWriter{
|
||||
Writer: fileDst,
|
||||
@@ -83,11 +85,6 @@ func monitorMain(options serveOptions) {
|
||||
}
|
||||
|
||||
args := os.Args
|
||||
binary, err := getBinary(args[0])
|
||||
if err != nil {
|
||||
l.Warnln("Error starting the main Syncthing process:", err)
|
||||
panic("Error starting the main Syncthing process")
|
||||
}
|
||||
var restarts [restartCounts]time.Time
|
||||
|
||||
stopSign := make(chan os.Signal, 1)
|
||||
@@ -109,7 +106,7 @@ func monitorMain(options serveOptions) {
|
||||
copy(restarts[0:], restarts[1:])
|
||||
restarts[len(restarts)-1] = time.Now()
|
||||
|
||||
cmd := exec.Command(binary, args[1:]...)
|
||||
cmd := exec.Command(args[0], args[1:]...)
|
||||
cmd.Env = childEnv
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
@@ -185,7 +182,7 @@ func monitorMain(options serveOptions) {
|
||||
// Restart the monitor process to release the .old
|
||||
// binary as part of the upgrade process.
|
||||
l.Infoln("Restarting monitor...")
|
||||
if err = restartMonitor(binary, args); err != nil {
|
||||
if err = restartMonitor(args); err != nil {
|
||||
l.Warnln("Restart:", err)
|
||||
}
|
||||
os.Exit(exitCode)
|
||||
@@ -208,19 +205,6 @@ func monitorMain(options serveOptions) {
|
||||
}
|
||||
}
|
||||
|
||||
func getBinary(args0 string) (string, error) {
|
||||
e, err := os.Executable()
|
||||
if err == nil {
|
||||
return e, nil
|
||||
}
|
||||
// Check if args0 cuts it
|
||||
e, lerr := exec.LookPath(args0)
|
||||
if lerr == nil {
|
||||
return e, nil
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
|
||||
func copyStderr(stderr io.Reader, dst io.Writer) {
|
||||
br := bufio.NewReader(stderr)
|
||||
|
||||
@@ -273,7 +257,7 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
|
||||
* This crash usually occurs due to one of the following reasons: *
|
||||
* - Syncthing being stopped abruptly (killed/loss of power) *
|
||||
* - Bad hardware (memory/disk issues) *
|
||||
* - Software that affects disk writes (SSD caching software and similar) *
|
||||
* - Software that affects disk writes (SSD caching software and simillar) *
|
||||
* *
|
||||
* Please see the following URL for instructions on how to recover: *
|
||||
* https://docs.syncthing.net/users/faq.html#my-syncthing-database-is-corrupt *
|
||||
@@ -327,30 +311,40 @@ func copyStdout(stdout io.Reader, dst io.Writer) {
|
||||
}
|
||||
}
|
||||
|
||||
func restartMonitor(binary string, args []string) error {
|
||||
func restartMonitor(args []string) error {
|
||||
// Set the STRESTART environment variable to indicate to the next
|
||||
// process that this is a restart and not initial start. This prevents
|
||||
// opening the browser on startup.
|
||||
os.Setenv("STRESTART", "yes")
|
||||
|
||||
if !build.IsWindows {
|
||||
if runtime.GOOS != "windows" {
|
||||
// syscall.Exec is the cleanest way to restart on Unixes as it
|
||||
// replaces the current process with the new one, keeping the pid and
|
||||
// controlling terminal and so on
|
||||
return restartMonitorUnix(binary, args)
|
||||
return restartMonitorUnix(args)
|
||||
}
|
||||
|
||||
// but it isn't supported on Windows, so there we start a normal
|
||||
// exec.Command and return.
|
||||
return restartMonitorWindows(binary, args)
|
||||
return restartMonitorWindows(args)
|
||||
}
|
||||
|
||||
func restartMonitorUnix(binary string, args []string) error {
|
||||
return syscall.Exec(binary, args, os.Environ())
|
||||
func restartMonitorUnix(args []string) error {
|
||||
if !strings.ContainsRune(args[0], os.PathSeparator) {
|
||||
// The path to the binary doesn't contain a slash, so it should be
|
||||
// found in $PATH.
|
||||
binary, err := exec.LookPath(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
args[0] = binary
|
||||
}
|
||||
|
||||
return syscall.Exec(args[0], args, os.Environ())
|
||||
}
|
||||
|
||||
func restartMonitorWindows(binary string, args []string) error {
|
||||
cmd := exec.Command(binary, args[1:]...)
|
||||
func restartMonitorWindows(args []string) error {
|
||||
cmd := exec.Command(args[0], args[1:]...)
|
||||
// Retain the standard streams
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Stdout = os.Stdout
|
||||
@@ -521,7 +515,7 @@ func (f *autoclosedFile) ensureOpenLocked() error {
|
||||
// We open the file for write only, and create it if it doesn't exist.
|
||||
flags := os.O_WRONLY | os.O_CREATE | os.O_APPEND
|
||||
|
||||
fd, err := os.OpenFile(f.name, flags, 0o644)
|
||||
fd, err := os.OpenFile(f.name, flags, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -570,7 +564,7 @@ func childEnv() []string {
|
||||
// panicUploadMaxWait uploading panics...
|
||||
func maybeReportPanics() {
|
||||
// Try to get a config to see if/where panics should be reported.
|
||||
cfg, err := loadOrDefaultConfig()
|
||||
cfg, err := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger, true)
|
||||
if err != nil {
|
||||
l.Warnln("Couldn't load config; not reporting crash")
|
||||
return
|
||||
|
||||
@@ -8,6 +8,7 @@ package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
@@ -17,17 +18,14 @@ import (
|
||||
func TestRotatedFile(t *testing.T) {
|
||||
// Verify that log rotation happens.
|
||||
|
||||
dir := t.TempDir()
|
||||
dir, err := ioutil.TempDir("", "syncthing")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
open := func(name string) (io.WriteCloser, error) {
|
||||
f, err := os.Create(name)
|
||||
t.Cleanup(func() {
|
||||
if f != nil {
|
||||
_ = f.Close()
|
||||
}
|
||||
})
|
||||
|
||||
return f, err
|
||||
return os.Create(name)
|
||||
}
|
||||
|
||||
logName := filepath.Join(dir, "log.txt")
|
||||
@@ -181,7 +179,7 @@ func TestAutoClosedFile(t *testing.T) {
|
||||
}
|
||||
|
||||
// The file should have both writes in it.
|
||||
bs, err := os.ReadFile(file)
|
||||
bs, err := ioutil.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -201,7 +199,7 @@ func TestAutoClosedFile(t *testing.T) {
|
||||
}
|
||||
|
||||
// It should now contain three writes, as the file is always opened for appending
|
||||
bs, err = os.ReadFile(file)
|
||||
bs, err = ioutil.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -4,25 +4,26 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
//go:build !windows
|
||||
// +build !windows
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"syscall"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/build"
|
||||
)
|
||||
|
||||
func openURL(url string) error {
|
||||
if build.IsDarwin {
|
||||
switch runtime.GOOS {
|
||||
case "darwin":
|
||||
return exec.Command("open", url).Run()
|
||||
|
||||
default:
|
||||
cmd := exec.Command("xdg-open", url)
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{
|
||||
Setpgid: true,
|
||||
}
|
||||
return cmd.Run()
|
||||
}
|
||||
cmd := exec.Command("xdg-open", url)
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{
|
||||
Setpgid: true,
|
||||
}
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
//go:build windows
|
||||
// +build windows
|
||||
|
||||
package main
|
||||
|
||||
@@ -4,11 +4,10 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
//go:build !windows
|
||||
// +build !windows
|
||||
|
||||
package cmdutil
|
||||
package main
|
||||
|
||||
type buildCommonOptions struct {
|
||||
type buildServeOptions struct {
|
||||
HideConsole bool `hidden:""`
|
||||
}
|
||||
@@ -4,8 +4,8 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package cmdutil
|
||||
package main
|
||||
|
||||
type buildCommonOptions struct {
|
||||
type buildServeOptions struct {
|
||||
HideConsole bool `name:"no-console" help:"Hide console window"`
|
||||
}
|
||||
@@ -4,7 +4,6 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
//go:build !solaris && !windows
|
||||
// +build !solaris,!windows
|
||||
|
||||
package main
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
//go:build solaris || windows
|
||||
// +build solaris windows
|
||||
|
||||
package main
|
||||
|
||||
@@ -4,8 +4,7 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
//go:build go1.7
|
||||
// +build go1.7
|
||||
//+build go1.7
|
||||
|
||||
package main
|
||||
|
||||
|
||||
@@ -4,11 +4,10 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package aggregate
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
@@ -16,21 +15,26 @@ import (
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
type CLI struct {
|
||||
DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"`
|
||||
var dbConn = getEnvDefault("UR_DB_URL", "postgres://user:password@localhost/ur?sslmode=disable")
|
||||
|
||||
func getEnvDefault(key, def string) string {
|
||||
if val := os.Getenv(key); val != "" {
|
||||
return val
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func (cli *CLI) Run() error {
|
||||
func main() {
|
||||
log.SetFlags(log.Ltime | log.Ldate)
|
||||
log.SetOutput(os.Stdout)
|
||||
|
||||
db, err := sql.Open("postgres", cli.DBConn)
|
||||
db, err := sql.Open("postgres", dbConn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("database: %w", err)
|
||||
log.Fatalln("database:", err)
|
||||
}
|
||||
err = setupDB(db)
|
||||
if err != nil {
|
||||
return fmt.Errorf("database: %w", err)
|
||||
log.Fatalln("database:", err)
|
||||
}
|
||||
|
||||
for {
|
||||
@@ -49,6 +53,13 @@ func runAggregation(db *sql.DB) {
|
||||
}
|
||||
log.Println("Inserted", rows, "rows")
|
||||
|
||||
log.Println("Aggregating UserMovement data")
|
||||
rows, err = aggregateUserMovement(db)
|
||||
if err != nil {
|
||||
log.Println("aggregate:", err)
|
||||
}
|
||||
log.Println("Inserted", rows, "rows")
|
||||
|
||||
since = maxIndexedDay(db, "Performance")
|
||||
log.Println("Aggregating Performance data since", since)
|
||||
rows, err = aggregatePerformance(db, since.Add(24*time.Hour))
|
||||
@@ -83,6 +94,16 @@ func setupDB(db *sql.DB) error {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS UserMovement (
|
||||
Day TIMESTAMP NOT NULL,
|
||||
Added INTEGER NOT NULL,
|
||||
Bounced INTEGER NOT NULL,
|
||||
Removed INTEGER NOT NULL
|
||||
)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Performance (
|
||||
Day TIMESTAMP NOT NULL,
|
||||
TotFiles INTEGER NOT NULL,
|
||||
@@ -98,13 +119,13 @@ func setupDB(db *sql.DB) error {
|
||||
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS BlockStats (
|
||||
Day TIMESTAMP NOT NULL,
|
||||
Reports INTEGER NOT NULL,
|
||||
Total BIGINT NOT NULL,
|
||||
Renamed BIGINT NOT NULL,
|
||||
Reused BIGINT NOT NULL,
|
||||
Pulled BIGINT NOT NULL,
|
||||
CopyOrigin BIGINT NOT NULL,
|
||||
CopyOriginShifted BIGINT NOT NULL,
|
||||
CopyElsewhere BIGINT NOT NULL
|
||||
Total INTEGER NOT NULL,
|
||||
Renamed INTEGER NOT NULL,
|
||||
Reused INTEGER NOT NULL,
|
||||
Pulled INTEGER NOT NULL,
|
||||
CopyOrigin INTEGER NOT NULL,
|
||||
CopyOriginShifted INTEGER NOT NULL,
|
||||
CopyElsewhere INTEGER NOT NULL
|
||||
)`)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -122,6 +143,11 @@ func setupDB(db *sql.DB) error {
|
||||
_, _ = db.Exec(`CREATE INDEX VersionDayIndex ON VersionSummary (Day)`)
|
||||
}
|
||||
|
||||
row = db.QueryRow(`SELECT 'MovementDayIndex'::regclass`)
|
||||
if err := row.Scan(&t); err != nil {
|
||||
_, _ = db.Exec(`CREATE INDEX MovementDayIndex ON UserMovement (Day)`)
|
||||
}
|
||||
|
||||
row = db.QueryRow(`SELECT 'PerformanceDayIndex'::regclass`)
|
||||
if err := row.Scan(&t); err != nil {
|
||||
_, _ = db.Exec(`CREATE INDEX PerformanceDayIndex ON Performance (Day)`)
|
||||
@@ -166,6 +192,87 @@ func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) {
|
||||
return res.RowsAffected()
|
||||
}
|
||||
|
||||
func aggregateUserMovement(db *sql.DB) (int64, error) {
|
||||
rows, err := db.Query(`SELECT
|
||||
DATE_TRUNC('day', Received) AS Day,
|
||||
Report->>'uniqueID'
|
||||
FROM ReportsJson
|
||||
WHERE
|
||||
Report->>'uniqueID' IS NOT NULL
|
||||
AND Received < DATE_TRUNC('day', NOW())
|
||||
AND Report->>'version' like 'v_.%'
|
||||
ORDER BY Day
|
||||
`)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
firstSeen := make(map[string]time.Time)
|
||||
lastSeen := make(map[string]time.Time)
|
||||
var minTs time.Time
|
||||
minTs = minTs.In(time.UTC)
|
||||
|
||||
for rows.Next() {
|
||||
var ts time.Time
|
||||
var id string
|
||||
if err := rows.Scan(&ts, &id); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if minTs.IsZero() {
|
||||
minTs = ts
|
||||
}
|
||||
if _, ok := firstSeen[id]; !ok {
|
||||
firstSeen[id] = ts
|
||||
}
|
||||
lastSeen[id] = ts
|
||||
}
|
||||
|
||||
type sumRow struct {
|
||||
day time.Time
|
||||
added int
|
||||
removed int
|
||||
bounced int
|
||||
}
|
||||
var sumRows []sumRow
|
||||
for t := minTs; t.Before(time.Now().Truncate(24 * time.Hour)); t = t.AddDate(0, 0, 1) {
|
||||
var added, removed, bounced int
|
||||
old := t.Before(time.Now().AddDate(0, 0, -30))
|
||||
for id, first := range firstSeen {
|
||||
last := lastSeen[id]
|
||||
if first.Equal(t) && last.Equal(t) && old {
|
||||
bounced++
|
||||
continue
|
||||
}
|
||||
if first.Equal(t) {
|
||||
added++
|
||||
}
|
||||
if last == t && old {
|
||||
removed++
|
||||
}
|
||||
}
|
||||
sumRows = append(sumRows, sumRow{t, added, removed, bounced})
|
||||
}
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if _, err := tx.Exec("DELETE FROM UserMovement"); err != nil {
|
||||
tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
for _, r := range sumRows {
|
||||
if _, err := tx.Exec("INSERT INTO UserMovement (Day, Added, Removed, Bounced) VALUES ($1, $2, $3, $4)", r.day, r.added, r.removed, r.bounced); err != nil {
|
||||
tx.Rollback()
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
return int64(len(sumRows)), tx.Commit()
|
||||
}
|
||||
|
||||
func aggregatePerformance(db *sql.DB, since time.Time) (int64, error) {
|
||||
res, err := db.Exec(`INSERT INTO Performance (
|
||||
SELECT
|
||||
@@ -199,13 +306,13 @@ func aggregateBlockStats(db *sql.DB, since time.Time) (int64, error) {
|
||||
SELECT
|
||||
DATE_TRUNC('day', Received) AS Day,
|
||||
COUNT(1) As Reports,
|
||||
SUM((Report->'blockStats'->>'total')::numeric)::bigint AS Total,
|
||||
SUM((Report->'blockStats'->>'renamed')::numeric)::bigint AS Renamed,
|
||||
SUM((Report->'blockStats'->>'reused')::numeric)::bigint AS Reused,
|
||||
SUM((Report->'blockStats'->>'pulled')::numeric)::bigint AS Pulled,
|
||||
SUM((Report->'blockStats'->>'copyOrigin')::numeric)::bigint AS CopyOrigin,
|
||||
SUM((Report->'blockStats'->>'copyOriginShifted')::numeric)::bigint AS CopyOriginShifted,
|
||||
SUM((Report->'blockStats'->>'copyElsewhere')::numeric)::bigint AS CopyElsewhere
|
||||
SUM((Report->'blockStats'->>'total')::numeric) AS Total,
|
||||
SUM((Report->'blockStats'->>'renamed')::numeric) AS Renamed,
|
||||
SUM((Report->'blockStats'->>'reused')::numeric) AS Reused,
|
||||
SUM((Report->'blockStats'->>'pulled')::numeric) AS Pulled,
|
||||
SUM((Report->'blockStats'->>'copyOrigin')::numeric) AS CopyOrigin,
|
||||
SUM((Report->'blockStats'->>'copyOriginShifted')::numeric) AS CopyOriginShifted,
|
||||
SUM((Report->'blockStats'->>'copyElsewhere')::numeric) AS CopyElsewhere
|
||||
FROM ReportsJson
|
||||
WHERE
|
||||
Received > $1
|
||||
@@ -4,7 +4,7 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package serve
|
||||
package main
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
@@ -145,7 +145,7 @@ func statsForFloats(data []float64) [4]float64 {
|
||||
return res
|
||||
}
|
||||
|
||||
func group(by func(string) string, as []analytic, perGroup int, otherPct float64) []analytic {
|
||||
func group(by func(string) string, as []analytic, perGroup int) []analytic {
|
||||
var res []analytic
|
||||
|
||||
next:
|
||||
@@ -170,25 +170,6 @@ next:
|
||||
}
|
||||
|
||||
sort.Sort(analyticList(res))
|
||||
|
||||
if otherPct > 0 {
|
||||
// Groups with less than otherPCt go into "Other"
|
||||
other := analytic{
|
||||
Key: "Other",
|
||||
}
|
||||
for i := 0; i < len(res); i++ {
|
||||
if res[i].Percentage < otherPct || res[i].Key == "Other" {
|
||||
other.Count += res[i].Count
|
||||
other.Percentage += res[i].Percentage
|
||||
res = append(res[:i], res[i+1:]...)
|
||||
i--
|
||||
}
|
||||
}
|
||||
if other.Count > 0 {
|
||||
res = append(res, other)
|
||||
}
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package serve
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package serve
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
1186
cmd/ursrv/main.go
1186
cmd/ursrv/main.go
File diff suppressed because it is too large
Load Diff
143
cmd/ursrv/migration.go
Normal file
143
cmd/ursrv/migration.go
Normal file
@@ -0,0 +1,143 @@
|
||||
// Copyright (C) 2020 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/lib/pq"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/ur/contract"
|
||||
)
|
||||
|
||||
func migrate(db *sql.DB) error {
|
||||
var count uint64
|
||||
log.Println("Checking old table row count, this might take a while...")
|
||||
if err := db.QueryRow(`SELECT COUNT(1) FROM Reports`).Scan(&count); err != nil || count == 0 {
|
||||
// err != nil most likely means table does not exist.
|
||||
return nil
|
||||
}
|
||||
log.Printf("Found %d records, will perform migration.", count)
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
log.Println("sql:", err)
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
// These must be lower case, because we don't quote them when creating, so postgres creates them lower case.
|
||||
// Yet pg.CopyIn quotes them, which makes them case sensitive.
|
||||
stmt, err := tx.Prepare(pq.CopyIn("reportsjson", "received", "report"))
|
||||
if err != nil {
|
||||
log.Println("sql:", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Custom types used in the old struct.
|
||||
var rep contract.Report
|
||||
var rescanIntvs pq.Int64Array
|
||||
var fsWatcherDelay pq.Int64Array
|
||||
pullOrder := make(IntMap)
|
||||
fileSystemType := make(IntMap)
|
||||
themes := make(IntMap)
|
||||
transportStats := make(IntMap)
|
||||
|
||||
rows, err := db.Query(`SELECT ` + strings.Join(rep.FieldNames(), ", ") + `, FolderFsWatcherDelays, RescanIntvs, FolderPullOrder, FolderFilesystemType, GUITheme, Transport FROM Reports`)
|
||||
if err != nil {
|
||||
log.Println("sql:", err)
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var done uint64
|
||||
pct := count / 100
|
||||
|
||||
for rows.Next() {
|
||||
err := rows.Scan(append(rep.FieldPointers(), &fsWatcherDelay, &rescanIntvs, &pullOrder, &fileSystemType, &themes, &transportStats)...)
|
||||
if err != nil {
|
||||
log.Println("sql scan:", err)
|
||||
return err
|
||||
}
|
||||
// Patch up parts that used to use custom types
|
||||
rep.RescanIntvs = make([]int, len(rescanIntvs))
|
||||
for i := range rescanIntvs {
|
||||
rep.RescanIntvs[i] = int(rescanIntvs[i])
|
||||
}
|
||||
rep.FolderUsesV3.FsWatcherDelays = make([]int, len(fsWatcherDelay))
|
||||
for i := range fsWatcherDelay {
|
||||
rep.FolderUsesV3.FsWatcherDelays[i] = int(fsWatcherDelay[i])
|
||||
}
|
||||
rep.FolderUsesV3.PullOrder = pullOrder
|
||||
rep.FolderUsesV3.FilesystemType = fileSystemType
|
||||
rep.GUIStats.Theme = themes
|
||||
rep.TransportStats = transportStats
|
||||
|
||||
_, err = stmt.Exec(rep.Received, rep)
|
||||
if err != nil {
|
||||
log.Println("sql insert:", err)
|
||||
return err
|
||||
}
|
||||
done++
|
||||
if done%pct == 0 {
|
||||
log.Printf("Migration progress %d/%d (%d%%)", done, count, (100*done)/count)
|
||||
}
|
||||
}
|
||||
|
||||
// Tell the driver bulk copy is finished
|
||||
_, err = stmt.Exec()
|
||||
if err != nil {
|
||||
log.Println("sql stmt exec:", err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = stmt.Close()
|
||||
if err != nil {
|
||||
log.Println("sql stmt close:", err)
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = tx.Exec("DROP TABLE Reports")
|
||||
if err != nil {
|
||||
log.Println("sql drop:", err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
log.Println("sql commit:", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type IntMap map[string]int
|
||||
|
||||
func (p IntMap) Value() (driver.Value, error) {
|
||||
return json.Marshal(p)
|
||||
}
|
||||
|
||||
func (p *IntMap) Scan(src interface{}) error {
|
||||
source, ok := src.([]byte)
|
||||
if !ok {
|
||||
return errors.New("Type assertion .([]byte) failed.")
|
||||
}
|
||||
|
||||
var i map[string]int
|
||||
err := json.Unmarshal(source, &i)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*p = i
|
||||
return nil
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
// Copyright (C) 2023 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package serve
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var metricReportsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "ursrv",
|
||||
Name: "reports_total",
|
||||
}, []string{"version"})
|
||||
|
||||
func init() {
|
||||
metricReportsTotal.WithLabelValues("fail")
|
||||
metricReportsTotal.WithLabelValues("duplicate")
|
||||
metricReportsTotal.WithLabelValues("v1")
|
||||
metricReportsTotal.WithLabelValues("v2")
|
||||
metricReportsTotal.WithLabelValues("v3")
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
Before Width: | Height: | Size: 4.8 KiB After Width: | Height: | Size: 4.8 KiB |
|
Before Width: | Height: | Size: 61 KiB After Width: | Height: | Size: 61 KiB |
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user