Compare commits

..

1 Commits

Author SHA1 Message Date
Jakob Borg
f3a793ce91 Add peer node sync status in GUI (fixes #46) 2014-02-16 08:30:32 +01:00
880 changed files with 12201 additions and 197714 deletions

View File

@@ -1,5 +0,0 @@
coverage:
range: "40...100"
ignore:
- "**.pb.go"

8
.gitattributes vendored
View File

@@ -1,8 +0,0 @@
# Text files use LF line endings in this repository
* text=auto
# Except the dependencies, which we leave alone
vendor/** -text=auto
# Diffs on these files are meaningless
*.svg -diff

2
.github/CODEOWNERS vendored
View File

@@ -1,2 +0,0 @@
/AUTHORS @calmh
/*.md @calmh

View File

@@ -1,42 +0,0 @@
### DO NOT REPORT SECURITY ISSUES IN THIS ISSUE TRACKER
Instead, contact security@syncthing.net directly - see
https://syncthing.net/security.html for more information.
### DO NOT POST SUPPORT REQUESTS OR GENERAL QUESTIONS IN THIS ISSUE TRACKER
Please use the forum at https://forum.syncthing.net/ where a large number of
helpful people hang out. This issue tracker is for reporting bugs or feature
requests directly to the developers. Worst case you might get a short
"that's a bug, please report it on GitHub" response on the forum, in which
case we thank you for your patience and following our advice. :)
### Please use the correct issue tracker
If your problem relates to a Syncthing wrapper or [sub-project](https://github.com/syncthing) such as [Syncthing for Android](https://github.com/syncthing/syncthing-android/issues), [SyncTrayzor](https://github.com/canton7/synctrayzor) or the [documentation](https://github.com/syncthing/docs/issues), please use their respective issue trackers.
### Does your log mention database corruption?
If your Syncthing log reports panics because of database corruption it is most likely a fault with your system's storage or memory. Affected log entries will contain lines starting with `panic: leveldb`. You will need to delete the index database to clear this, by running `syncthing -reset-database`.
### Please do post actual bug reports and feature requests.
If your issue is a bug report, replace this boilerplate with a description
of the problem, being sure to include at least:
- what happened,
- what you expected to happen instead, and
- any steps to reproduce the problem.
Also fill out the version information below and add log output or
screenshots as appropriate.
If your issue is a feature request, simply replace this template text in
its entirety.
### Version Information
Syncthing Version: v0.x.y
OS Version: Windows 7 / Ubuntu 14.04 / ...
Browser Version: (if applicable, for GUI issues)

View File

@@ -1,27 +0,0 @@
### Purpose
Describe the purpose of this change. If there is an existing issue that is
resolved by this pull request, ensure that the commit subject is on the form
`Some short description (fixes #1234)` where 1234 is the issue number.
### Testing
Describe what testing has been done, and how the reviewer can test the change
if new tests are not included.
### Screenshots
If this is a GUI change, include screenshots of the change. If not, please
feel free to just delete this section.
### Documentation
If this is a user visible change (including API and protocol changes), add a link here
to the corresponding pull request on https://github.com/syncthing/docs or describe
the documentation changes necessary.
## Authorship
Your name and email will be added automatically to the AUTHORS file
based on the commit metadata.

48
.github/SECURITY.md vendored
View File

@@ -1,48 +0,0 @@
## Reporting a Vulnerability
If you believe that you've found a Syncthing-related security vulnerability, please report it by sending email to the address security@syncthing.net. The PGP key for security@syncthing.net (B683AD7B76CAB013) below can be used to send encrypted mail or to verify responses received from that address.
```
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1
mQENBFShFlIBCACsW346HYskhKhxrdZMjyU5Ntsjvg6/ogqINDPoL10/oaIP0G+t
7zzC0K5Cq29ix43kNNLKTJNyPkdTeaJEcqslMUt6tovjHwoKJ073GP0W3KsNvBRg
ffCZOAexGfOsBSL9KHaYGK67Py3TFgtN1H/EmboU1arrLfAMrmqOip++EGqOxjse
gH0qk7Mk1TqEC5Xh3NGE7r1UobAlqdUv5E3v7U11NhAdP1zu/XZ/zvP5mwVQJMLv
iZyeWGliNI8nEeRjYw+S85f4gq7H2mgoeNBN4WwwK1hhz9qpvCsgPW3XqlExTPI4
1vM4PxpiFIuF0zuy2OwsmjrpTCZeBscr4Tj5ABEBAAG0K1N5bmN0aGluZyBTZWN1
cml0eSA8c2VjdXJpdHlAc3luY3RoaW5nLm5ldD6JATgEEwECACIFAlShFlICGwMG
CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELaDrXt2yrATB1UH/jnnIa6DekCA
2V36N/2+pFSNLVWeoZQxZ9ne9S7WSubaoK4oCWiuChPSAy5hagnKnNA3a9wrz5iN
0hgCA88NnTU18biZW6HX8xWEd3dnY3fX1sG0XdHuKlFria8ByfcrbShf/CttXkEd
Y5qPHH9aLIMtksBS1MIsRYrOCHiLNYFCKlbjDDCGT3tuk/yaU7aBAFVPIag54afC
zZAIvTIgRruLWkNT/sSEJx9ekhJzO/+pXNbSSHwhoj5OJh7sQjZWwPzNazelk48R
+ku8VpB5Wxgk2MnJPf1RxF+7saBAaeAVZsJcPN+Jp7u9LCFelIRn1ISsHbhLhyqL
cPXqllltLCKJAhwEEAECAAYFAlShFlsACgkQSfWuwLzlJMcEpA//VvZYHGZgZMM0
bBkYnjManYnJkHSQ2pHsjquSFH+fbfq2FxxTk/nQ/IAvBzt8NDTT3ylPOmsl4A8q
r8BxsRNENhU7zDQrKC5NtYUrzXhPGo8qfDwkeLyqd2msUvHn7EH3PNgiN2QaFWxE
21FqoXIpERKJRgRtv1SYZoMyNGjmT2hta1ZbwEfrPJnzjYhneoUGsRApG7p+uPqe
4LRpbG3Fa2vBm/UWUrOe+6jPzvMokjIJbdK0IjXamFAzwYW5fSZaFa94mR6L5f5m
X8Dhp66mBRTx7qk6ldEqptvaYGqihaWP1xbDaApQsGHQujtcBYGp+edlkabuW5h7
stl/7QTFkEPqHT4ybxEX0rLoFUGq56WUlKp27z40keStaXMgfrsxJtkz66Xs8vU4
7qZcLAcPsin+y0toqavtwtM/L7uCMu1yhbFRGJ+JL6saJAqzwS8l7r6E2R7OnVj1
BdASgxx1TgzW6ZFW5p1Iy6mtpkBypsnp8s3UcP3GFRnQe9gi1EXHjzuZAUGafe4Q
juvJ8t8xIcQMFuAylNIVyXvIWJoehqsVY9EBgVtE4y1SRh8XTT3Tddn8ab5fl7uh
HWAY8cRlv6WIOhK8w8oroiYx0SID8jMeEwJBS4DL7qWtJMkDo8ZEJiB+Pkd963MX
05QXt33AvJJ9PmbGCHkcH7198tCmA+e5AQ0EVKEWUgEIAPGczHpa6NdxY0pm+tIp
btiA6gdPE70pJYgJTKX7siCQ2w770H3CBSKmqEXadr7WnfIgUYIDaSxadeGzB/Mn
3SHCYRCqbA7mwu2k4wNNvCEM0xZqFAvaDJ4avlZ5oiMT8IFHKsjC77nkhmfXaIGt
hn10H2MFADjvJYul7vR+Ghg1wGhTGWo2u7VVj9BI+AfvnWaouFI0cx2KNWEI/Ocj
z6jk8nmC3yOEFQECM/hF4lkAOv9CQUa8UcvAr31trzawmV1iSsKjmVZgqd0N4T8f
hUikqUPZGNCRcqEUffTzggIyGPbedFnZw9Di7o1xByxyTrZycemAVqaVGF+9nFLG
pccAEQEAAYkBHwQYAQIACQUCVKEWUgIbDAAKCRC2g617dsqwEzrjB/9q0T8A4XUE
p0g6xq86jhmh4jlEedxrfXUL3R6ejFtuKMThulxEP0xiQ7xLzBhOnvyxLCsVbjp8
0JtJTVCq44UzUiIBuVNRoYG29uXuTUL2UtI27VhjFxMzLDwZL97tbGR6lzdM/+U5
9PC+PIvS0yz13z2t3/x3KUOnBgxZnpy9h4AdKwjrNVtnQdGDXSKlJBLb57TFcS94
f3roZ5Gpw3AWYSmSSiZWbhks2UNzYSQ84LAKlV1NkktO+qRc/pUqr6IxMjOKc7XP
e8u1Nst6fN3GNqZOV+jUYs/fqrJgp1TUWjNTuf22Rl0Idz7XLPJKYFh9W7T/4MbU
M7Q8GZuww1rk
=No/v
-----END PGP PUBLIC KEY BLOCK-----
```

27
.gitignore vendored
View File

@@ -1,26 +1,3 @@
/syncthing
/stdiscosrv
syncthing.exe
stdiscosrv.exe
syncthing
*.tar.gz
*.zip
*.asc
*.deb
.jshintrc
coverage.out
files/pidx
bin
perfstats*.csv
coverage.xml
syncthing.sig
RELEASE
deb
lib/auto/gui.files.go
snapcraft.yaml
prime/
snap/
parts/
stage/
*.snap
*.bz2
/repos
dist

View File

@@ -1,20 +0,0 @@
linters-settings:
maligned:
suggest-new: true
linters:
enable-all: true
disable:
- goimports
- depguard
- lll
- gochecknoinits
- gochecknoglobals
- gofmt
- scopelint
service:
golangci-lint-version: 1.16.x
prepare:
- GO111MODULE=on go mod vendor
- go run build.go assets

208
AUTHORS
View File

@@ -1,208 +0,0 @@
# This is the official list of Syncthing authors for copyright purposes.
#
# THIS FILE IS MOSTLY AUTO GENERATED. IF YOU'VE MADE A COMMIT TO THE
# REPOSITORY YOU WILL BE ADDED HERE AUTOMATICALLY WITHOUT THE NEED FOR
# ANY MANUAL ACTION.
#
# That said, you are welcome to correct your name or add a nickname / GitHub
# user name as appropriate. The format is:
#
# Name Name Name (nickname) <email1@example.com> <email2@example.com>
#
# The in-GUI authors list is periodically automatically updated from the
# contents of this file.
#
Aaron Bieber (qbit) <qbit@deftly.net>
Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com> <adam@proactiveservices.co.uk>
Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
Alexander Graf (alex2108) <register-github@alex-graf.de>
Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
andresvia <andres.via@gmail.com>
Andrew Dunham (andrew-d) <andrew@du.nham.ca>
Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
andyleap <andyleap@gmail.com>
Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
Antony Male (canton7) <antony.male@gmail.com>
Aranjedeath <Aranjedeath@users.noreply.github.com>
Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
Bart De Vries (mogwa1) <devriesb@gmail.com>
Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
Ben Shepherd (benshep) <bjashepherd@gmail.com>
Ben Sidhom (bsidhom) <bsidhom@gmail.com>
Benedikt Heine (bebehei) <bebe@bebehei.de>
Benedikt Morbach <benedikt.morbach@googlemail.com>
Benno Fünfstück <benno.fuenfstueck@gmail.com>
Benny Ng (tpng) <benny.tpng@gmail.com>
Boris Rybalkin <ribalkin@gmail.com>
Brandon Philips (philips) <brandon@ifup.org>
Brendan Long (brendanlong) <self@brendanlong.com>
Brian R. Becker (brbecker) <brbecker@gmail.com>
Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
Cedric Staniewski (xduugu) <cedric@gmx.ca>
Chris Howie (cdhowie) <me@chrishowie.com>
Chris Joel (cdata) <chris@scriptolo.gy>
Chris Tonkinson <chris@masterbran.ch>
chucic <chucic@seznam.cz>
Colin Kennedy (moshen) <moshen.colin@gmail.com>
Cromefire_ <tim.l@nghorst.net>
Dale Visser <dale.visser@live.com>
Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
Daniel Martí (mvdan) <mvdan@mvdan.cc>
Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
Denis A. (dva) <denisva@gmail.com>
Dennis Wilson (snnd) <dw@risu.io>
dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com>
dependabot[bot] <dependabot[bot]@users.noreply.github.com>
derekriemer <derek.riemer@colorado.edu>
desbma <desbma@users.noreply.github.com>
Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
Dominik Heidler (asdil12) <dominik@heidler.eu>
Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
Elliot Huffman <thelich2@gmail.com>
Emil Hessman (ceh) <emil@hessman.se>
Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
Evgeny Kuznetsov <evgeny@kuznetsov.md>
Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
Felix Ableitner (Nutomic) <me@nutomic.com>
Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
Frank Isemann (fti7) <frank@isemann.name>
georgespatton <georgespatton@users.noreply.github.com>
Gilli Sigurdsson (gillisig) <gilli@vx.is>
Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
Han Boetes <han@boetes.org>
Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
Heiko Zuerker (Smiley73) <heiko@zuerker.org>
Hugo Locurcio <hugo.locurcio@hugo.pro>
Iain Barnett <iainspeed@gmail.com>
Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
Iskander Sharipov (Alex) <quasilyte@gmail.com>
Jaakko Hannikainen (jgke) <jgke@jgke.fi>
Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
Jake Peterson (acogdev) <jake@acogdev.com>
Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
janost <janost@tuta.io>
Jaroslav Malec (dzarda) <dzardacz@gmail.com>
jaseg <githubaccount@jaseg.net>
Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
Jochen Voss (seehuhn) <voss@seehuhn.de>
Johan Andersson <j@i19.se>
Johan Vromans (sciurius) <jvromans@squirrel.nl>
John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
Jonas Thelemann <e-mail@jonas-thelemann.de>
Jonathan Cross <jcross@gmail.com>
Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
Jörg Thalheim <Mic92@users.noreply.github.com>
Kalle Laine <pahakalle@protonmail.com>
Karol Różycki (krozycki) <rozycki.karol@gmail.com>
Keith Turner <kturner@apache.org>
Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
Kevin Allen (ironmig) <kma1660@gmail.com>
Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
klemens <ka7@github.com>
Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
Laurent Arnoud <laurent@spkdev.net>
Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
Leo Arias (elopio) <yo@elopio.net>
Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
Lode Hoste (Zillode) <zillode@zillode.be>
Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
Marc Pujol (kilburn) <kilburn@la3.org>
Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
marco-m <marco.molteni@laposte.net>
Mark Pulford (mpx) <mark@kyne.com.au>
Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
Matic Potočnik <hairyfotr@gmail.com>
Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
Matt Robenolt <matt@ydekproductions.com>
Matteo Ruina <matteo.ruina@gmail.com>
Maurizio Tomasi <ziotom78@gmail.com>
Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
MaximAL <almaximal@ya.ru>
Maxime Thirouin <m@moox.io>
Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
Michael Ploujnikov (plouj) <ploujj@gmail.com>
Michael Tilli (pyfisch) <pyfisch@gmail.com>
Mike Boone <mike@boonedocks.net>
MikeLund <MikeLund@users.noreply.github.com>
Mingxuan Lin <gdlmx@users.noreply.github.com>
Nate Morrison (nrm21) <natemorrison@gmail.com>
Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
Nils Jakobi (thunderstorm99) <jakobi.nils@gmail.com>
Nitroretro <43112364+Nitroretro@users.noreply.github.com>
NoLooseEnds <jon.koslung@gmail.com>
otbutz <tbutz@optitool.de>
Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
Paweł Rozlach <vespian@users.noreply.github.com>
perewa <cavalcante.ten@gmail.com>
Peter Badida <KeyWeeUsr@users.noreply.github.com>
Peter Dave Hello <hsu@peterdavehello.org>
Peter Hoeg (peterhoeg) <peter@speartail.com>
Peter Marquardt (wwwutz) <wwwutz@gmail.com> <wwwutz@googlemail.com>
Phil Davis <phil.davis@inf.org>
Philippe Schommers (filoozoom) <philippe@schommers.be>
Phill Luby (pluby) <phill.luby@newredo.com>
Pier Paolo Ramon <ramonpierre@gmail.com>
Piotr Bejda (piobpl) <piotrb10@gmail.com>
Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
Richard Hartmann <RichiH@users.noreply.github.com>
Robert Carosi (nov1n) <robert@carosi.nl>
Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
Ross Smith II (rasa) <ross@smithii.com>
rubenbe <github-com-00ff86@vandamme.email>
Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
Scott Klupfel (kluppy) <kluppy@going2blue.com>
Sergey Mishin (ralder) <ralder@yandex.ru>
Simon Frei (imsodin) <freisim93@gmail.com>
Sly_tom_cat <slytomcat@mail.ru>
Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
Thomas Hipp <thomashipp@gmail.com>
Tim Abell (timabell) <tim@timwise.co.uk>
Tim Howes (timhowes) <timhowes@berkeley.edu>
Tobias Nygren (tnn2) <tnn@nygren.pp.se>
Tobias Tom (tobiastom) <t.tom@succont.de>
Tom Jakubowski <tom@crystae.net>
Tomas Cerveny (kozec) <kozec@kozec.com>
Tommy Thorn <tommy-github-email@thorn.ws>
Tully Robinson (tojrobinson) <tully@tojr.org>
Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
Victor Buinsky (buinsky) <vix_booja@tut.by>
Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
Vladimir Rusinov <vrusinov@google.com>
wangguoliang <liangcszzu@163.com>
William A. Kennington III (wkennington) <william@wkennington.com>
Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
Xavier O. (damajor) <damajor@gmail.com>
xjtdy888 (xjtdy888) <xjtdy888@163.com>
Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
佛跳墙 <daoquan@qq.com>

View File

@@ -1,73 +0,0 @@
# Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, gender identity and expression, level of experience,
education, socio-economic status, nationality, personal appearance, race,
religion, or sexual identity and orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at security@syncthing.net. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
[homepage]: https://www.contributor-covenant.org

View File

@@ -1,63 +1,22 @@
## Reporting Bugs
Please do contribute!
Please file bugs in the [Github Issue
Tracker](https://github.com/syncthing/syncthing/issues). Include at
least the following:
## Building
- What happened
[See the wiki](https://github.com/calmh/syncthing/wiki/Building)
- What did you expect to happen instead of what *did* happen, if it's
not crazy obvious
## Tests
- What operating system, operating system version and version of
Syncthing you are running
Yes please!
- The same for other connected devices, where relevant
## Style
- Screenshot if the issue concerns something visible in the GUI
`go fmt`
- Console log entries, where possible and relevant
## Documentation
If you're not sure whether something is relevant, erring on the side of
too much information will never get you yelled at. :)
[Hack it here](https://github.com/calmh/syncthing/wiki)
## Contributing Translations
## License
All translations are done via
[Transifex](https://www.transifex.com/projects/p/syncthing/). If you
wish to contribute to a translation, just head over there and sign up.
Before every release, the language resources are updated from the
latest info on Transifex.
## Contributing Code
Every contribution is welcome. If you want to contribute but are unsure
where to start, any open issues are fair game! See the [Contribution
Guidelines](https://docs.syncthing.net/dev/contributing.html) for the full
story on committing code.
## Contributing Documentation
Updates to the [documentation site](https://docs.syncthing.net/) can be
made as pull requests on the [documentation
repository](https://github.com/syncthing/docs).
## Licensing
All contributions are made available under the same license as the already
existing material being contributed to. For most of the project and unless
otherwise stated this means MPLv2, but there are exceptions:
- Certain commands (under cmd/...) may have a separate license, indicated by
the presence of a LICENSE file in the corresponding directory.
- The documentation (man/...) is licensed under the Creative Commons
Attribution 4.0 International License.
- Projects under vendor/... are copyright by and licensed from their
respective original authors. Contributions should be made to the original
project, not here.
Regardless of the license in effect, you retain the copyright to your
contribution.
MIT

View File

@@ -1,27 +0,0 @@
FROM golang:1.12 AS builder
WORKDIR /src
COPY . .
ENV CGO_ENABLED=0
ENV BUILD_HOST=syncthing.net
ENV BUILD_USER=docker
RUN rm -f syncthing && go run build.go -no-upgrade build syncthing
FROM alpine
EXPOSE 8384 22000 21027/udp
VOLUME ["/var/syncthing"]
RUN apk add --no-cache ca-certificates su-exec
COPY --from=builder /src/syncthing /bin/syncthing
COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
ENV PUID=1000 PGID=1000
HEALTHCHECK --interval=1m --timeout=10s \
CMD nc -z localhost 8384 || exit 1
ENTRYPOINT ["/bin/entrypoint.sh", "-home", "/var/syncthing/config", "-gui-address", "0.0.0.0:8384"]

View File

@@ -1,83 +0,0 @@
# The Syncthing Goals
Syncthing is a **continuous file synchronization program**. It synchronizes
files between two or more computers. We strive to fulfill the goals below.
The goals are listed in order of importance, the most important one being
the first.
> "Syncing files" here is precise. It means we specifically exclude things
> that are not files - calendar items, instant messages, and so on. If those
> are in fact stored as files on disk, they can of course be synced as
> files.
Syncthing should be:
### 1. Safe From Data Loss
Protecting the user's data is paramount. We take every reasonable precaution
to avoid corrupting the user's files.
> This is the overriding goal, without which synchronizing files becomes
> pointless. This means that we do not make unsafe trade offs for the sake
> of performance or, in some cases, even usability.
### 2. Secure Against Attackers
Again, protecting the user's data is paramount. Regardless of our other
goals we must never allow the user's data to be susceptible to eavesdropping
or modification by unauthorized parties.
> This should be understood in context. It is not necessarily reasonable to
> expect Syncthing to be resistant against well equipped state level
> attackers. We will however do our best. Note also that this is different
> from anonymity which is not, currently, a goal.
### 3. Easy to Use
Syncthing should be approachable, understandable and inclusive.
> Complex concepts and maths form the base of Syncthing's functionality.
> This should nonetheless be abstracted or hidden to a degree where
> Syncthing is usable by the general public.
### 4. Automatic
User interaction should be required only when absolutely necessary.
> Specifically this means that changes to files are picked up without
> prompting, conflicts are resolved without prompting and connections are
> maintained without prompting. We only prompt the user when it is required
> to fulfill one of the (overriding) Secure, Safe or Easy goals.
### 5. Universally Available
Syncthing should run on every common computer. We are mindful that the
latest technology is not always available to any given individual.
> Computers include desktops, laptops, servers, virtual machines, small
> general purpose computers such as Raspberry Pis and, *where possible*,
> tablets and phones. NAS appliances, toasters, cars, firearms, thermostats
> and so on may include computing capabitilies but it is not our goal for
> Syncthing to run smoothly on these devices.
### 6. For Individuals
Syncthing is primarily about empowering the individual user with safe,
secure and easy to use file synchronization.
> We acknowledge that it's also useful in an enterprise setting and include
> functionality to support that. If this is in conflict with the
> requirements of the individual, those will however take priority.
### 7. Everything Else
There are many things we care about that don't make it on to the list. It is
fine to optimize for these values as well, as long as they are not in
conflict with the stated goals above.
> For example, performance is a thing we care about. We just don't care more
> about it than safety, security, etc. Maintainability of the code base and
> providing entertainment value for the maintainers are also things that
> matter. It is understood that there are aspects of Syncthing that are
> suboptimal or even in opposition with the goals above. However, we
> continuously strive to align Syncthing more and more with these goals.

392
LICENSE
View File

@@ -1,373 +1,19 @@
Mozilla Public License Version 2.0
==================================
1. Definitions
--------------
1.1. "Contributor"
means each individual or legal entity that creates, contributes to
the creation of, or owns Covered Software.
1.2. "Contributor Version"
means the combination of the Contributions of others (if any) used
by a Contributor and that particular Contributor's Contribution.
1.3. "Contribution"
means Covered Software of a particular Contributor.
1.4. "Covered Software"
means Source Code Form to which the initial Contributor has attached
the notice in Exhibit A, the Executable Form of such Source Code
Form, and Modifications of such Source Code Form, in each case
including portions thereof.
1.5. "Incompatible With Secondary Licenses"
means
(a) that the initial Contributor has attached the notice described
in Exhibit B to the Covered Software; or
(b) that the Covered Software was made available under the terms of
version 1.1 or earlier of the License, but not also under the
terms of a Secondary License.
1.6. "Executable Form"
means any form of the work other than Source Code Form.
1.7. "Larger Work"
means a work that combines Covered Software with other material, in
a separate file or files, that is not Covered Software.
1.8. "License"
means this document.
1.9. "Licensable"
means having the right to grant, to the maximum extent possible,
whether at the time of the initial grant or subsequently, any and
all of the rights conveyed by this License.
1.10. "Modifications"
means any of the following:
(a) any file in Source Code Form that results from an addition to,
deletion from, or modification of the contents of Covered
Software; or
(b) any new file in Source Code Form that contains any Covered
Software.
1.11. "Patent Claims" of a Contributor
means any patent claim(s), including without limitation, method,
process, and apparatus claims, in any patent Licensable by such
Contributor that would be infringed, but for the grant of the
License, by the making, using, selling, offering for sale, having
made, import, or transfer of either its Contributions or its
Contributor Version.
1.12. "Secondary License"
means either the GNU General Public License, Version 2.0, the GNU
Lesser General Public License, Version 2.1, the GNU Affero General
Public License, Version 3.0, or any later versions of those
licenses.
1.13. "Source Code Form"
means the form of the work preferred for making modifications.
1.14. "You" (or "Your")
means an individual or a legal entity exercising rights under this
License. For legal entities, "You" includes any entity that
controls, is controlled by, or is under common control with You. For
purposes of this definition, "control" means (a) the power, direct
or indirect, to cause the direction or management of such entity,
whether by contract or otherwise, or (b) ownership of more than
fifty percent (50%) of the outstanding shares or beneficial
ownership of such entity.
2. License Grants and Conditions
--------------------------------
2.1. Grants
Each Contributor hereby grants You a world-wide, royalty-free,
non-exclusive license:
(a) under intellectual property rights (other than patent or trademark)
Licensable by such Contributor to use, reproduce, make available,
modify, display, perform, distribute, and otherwise exploit its
Contributions, either on an unmodified basis, with Modifications, or
as part of a Larger Work; and
(b) under Patent Claims of such Contributor to make, use, sell, offer
for sale, have made, import, and otherwise transfer either its
Contributions or its Contributor Version.
2.2. Effective Date
The licenses granted in Section 2.1 with respect to any Contribution
become effective for each Contribution on the date the Contributor first
distributes such Contribution.
2.3. Limitations on Grant Scope
The licenses granted in this Section 2 are the only rights granted under
this License. No additional rights or licenses will be implied from the
distribution or licensing of Covered Software under this License.
Notwithstanding Section 2.1(b) above, no patent license is granted by a
Contributor:
(a) for any code that a Contributor has removed from Covered Software;
or
(b) for infringements caused by: (i) Your and any other third party's
modifications of Covered Software, or (ii) the combination of its
Contributions with other software (except as part of its Contributor
Version); or
(c) under Patent Claims infringed by Covered Software in the absence of
its Contributions.
This License does not grant any rights in the trademarks, service marks,
or logos of any Contributor (except as may be necessary to comply with
the notice requirements in Section 3.4).
2.4. Subsequent Licenses
No Contributor makes additional grants as a result of Your choice to
distribute the Covered Software under a subsequent version of this
License (see Section 10.2) or under the terms of a Secondary License (if
permitted under the terms of Section 3.3).
2.5. Representation
Each Contributor represents that the Contributor believes its
Contributions are its original creation(s) or it has sufficient rights
to grant the rights to its Contributions conveyed by this License.
2.6. Fair Use
This License is not intended to limit any rights You have under
applicable copyright doctrines of fair use, fair dealing, or other
equivalents.
2.7. Conditions
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
in Section 2.1.
3. Responsibilities
-------------------
3.1. Distribution of Source Form
All distribution of Covered Software in Source Code Form, including any
Modifications that You create or to which You contribute, must be under
the terms of this License. You must inform recipients that the Source
Code Form of the Covered Software is governed by the terms of this
License, and how they can obtain a copy of this License. You may not
attempt to alter or restrict the recipients' rights in the Source Code
Form.
3.2. Distribution of Executable Form
If You distribute Covered Software in Executable Form then:
(a) such Covered Software must also be made available in Source Code
Form, as described in Section 3.1, and You must inform recipients of
the Executable Form how they can obtain a copy of such Source Code
Form by reasonable means in a timely manner, at a charge no more
than the cost of distribution to the recipient; and
(b) You may distribute such Executable Form under the terms of this
License, or sublicense it under different terms, provided that the
license for the Executable Form does not attempt to limit or alter
the recipients' rights in the Source Code Form under this License.
3.3. Distribution of a Larger Work
You may create and distribute a Larger Work under terms of Your choice,
provided that You also comply with the requirements of this License for
the Covered Software. If the Larger Work is a combination of Covered
Software with a work governed by one or more Secondary Licenses, and the
Covered Software is not Incompatible With Secondary Licenses, this
License permits You to additionally distribute such Covered Software
under the terms of such Secondary License(s), so that the recipient of
the Larger Work may, at their option, further distribute the Covered
Software under the terms of either this License or such Secondary
License(s).
3.4. Notices
You may not remove or alter the substance of any license notices
(including copyright notices, patent notices, disclaimers of warranty,
or limitations of liability) contained within the Source Code Form of
the Covered Software, except that You may alter any license notices to
the extent required to remedy known factual inaccuracies.
3.5. Application of Additional Terms
You may choose to offer, and to charge a fee for, warranty, support,
indemnity or liability obligations to one or more recipients of Covered
Software. However, You may do so only on Your own behalf, and not on
behalf of any Contributor. You must make it absolutely clear that any
such warranty, support, indemnity, or liability obligation is offered by
You alone, and You hereby agree to indemnify every Contributor for any
liability incurred by such Contributor as a result of warranty, support,
indemnity or liability terms You offer. You may include additional
disclaimers of warranty and limitations of liability specific to any
jurisdiction.
4. Inability to Comply Due to Statute or Regulation
---------------------------------------------------
If it is impossible for You to comply with any of the terms of this
License with respect to some or all of the Covered Software due to
statute, judicial order, or regulation then You must: (a) comply with
the terms of this License to the maximum extent possible; and (b)
describe the limitations and the code they affect. Such description must
be placed in a text file included with all distributions of the Covered
Software under this License. Except to the extent prohibited by statute
or regulation, such description must be sufficiently detailed for a
recipient of ordinary skill to be able to understand it.
5. Termination
--------------
5.1. The rights granted under this License will terminate automatically
if You fail to comply with any of its terms. However, if You become
compliant, then the rights granted under this License from a particular
Contributor are reinstated (a) provisionally, unless and until such
Contributor explicitly and finally terminates Your grants, and (b) on an
ongoing basis, if such Contributor fails to notify You of the
non-compliance by some reasonable means prior to 60 days after You have
come back into compliance. Moreover, Your grants from a particular
Contributor are reinstated on an ongoing basis if such Contributor
notifies You of the non-compliance by some reasonable means, this is the
first time You have received notice of non-compliance with this License
from such Contributor, and You become compliant prior to 30 days after
Your receipt of the notice.
5.2. If You initiate litigation against any entity by asserting a patent
infringement claim (excluding declaratory judgment actions,
counter-claims, and cross-claims) alleging that a Contributor Version
directly or indirectly infringes any patent, then the rights granted to
You by any and all Contributors for the Covered Software under Section
2.1 of this License shall terminate.
5.3. In the event of termination under Sections 5.1 or 5.2 above, all
end user license agreements (excluding distributors and resellers) which
have been validly granted by You or Your distributors under this License
prior to termination shall survive termination.
************************************************************************
* *
* 6. Disclaimer of Warranty *
* ------------------------- *
* *
* Covered Software is provided under this License on an "as is" *
* basis, without warranty of any kind, either expressed, implied, or *
* statutory, including, without limitation, warranties that the *
* Covered Software is free of defects, merchantable, fit for a *
* particular purpose or non-infringing. The entire risk as to the *
* quality and performance of the Covered Software is with You. *
* Should any Covered Software prove defective in any respect, You *
* (not any Contributor) assume the cost of any necessary servicing, *
* repair, or correction. This disclaimer of warranty constitutes an *
* essential part of this License. No use of any Covered Software is *
* authorized under this License except under this disclaimer. *
* *
************************************************************************
************************************************************************
* *
* 7. Limitation of Liability *
* -------------------------- *
* *
* Under no circumstances and under no legal theory, whether tort *
* (including negligence), contract, or otherwise, shall any *
* Contributor, or anyone who distributes Covered Software as *
* permitted above, be liable to You for any direct, indirect, *
* special, incidental, or consequential damages of any character *
* including, without limitation, damages for lost profits, loss of *
* goodwill, work stoppage, computer failure or malfunction, or any *
* and all other commercial damages or losses, even if such party *
* shall have been informed of the possibility of such damages. This *
* limitation of liability shall not apply to liability for death or *
* personal injury resulting from such party's negligence to the *
* extent applicable law prohibits such limitation. Some *
* jurisdictions do not allow the exclusion or limitation of *
* incidental or consequential damages, so this exclusion and *
* limitation may not apply to You. *
* *
************************************************************************
8. Litigation
-------------
Any litigation relating to this License may be brought only in the
courts of a jurisdiction where the defendant maintains its principal
place of business and such litigation shall be governed by laws of that
jurisdiction, without reference to its conflict-of-law provisions.
Nothing in this Section shall prevent a party's ability to bring
cross-claims or counter-claims.
9. Miscellaneous
----------------
This License represents the complete agreement concerning the subject
matter hereof. If any provision of this License is held to be
unenforceable, such provision shall be reformed only to the extent
necessary to make it enforceable. Any law or regulation which provides
that the language of a contract shall be construed against the drafter
shall not be used to construe this License against a Contributor.
10. Versions of the License
---------------------------
10.1. New Versions
Mozilla Foundation is the license steward. Except as provided in Section
10.3, no one other than the license steward has the right to modify or
publish new versions of this License. Each version will be given a
distinguishing version number.
10.2. Effect of New Versions
You may distribute the Covered Software under the terms of the version
of the License under which You originally received the Covered Software,
or under the terms of any subsequent version published by the license
steward.
10.3. Modified Versions
If you create software not governed by this License, and you want to
create a new license for such software, you may create and use a
modified version of this License if you rename the license and remove
any references to the name of the license steward (except to note that
such modified license differs from this License).
10.4. Distributing Source Code Form that is Incompatible With Secondary
Licenses
If You choose to distribute Source Code Form that is Incompatible With
Secondary Licenses under the terms of this version of the License, the
notice described in Exhibit B of this License must be attached.
Exhibit A - Source Code Form License Notice
-------------------------------------------
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at https://mozilla.org/MPL/2.0/.
If it is not possible or desirable to put the notice in a particular
file, then You may include the notice in a location (such as a LICENSE
file in a relevant directory) where a recipient would be likely to look
for such a notice.
You may add additional accurate notices of copyright ownership.
Exhibit B - "Incompatible With Secondary Licenses" Notice
---------------------------------------------------------
This Source Code Form is "Incompatible With Secondary Licenses", as
defined by the Mozilla Public License, v. 2.0.
Copyright (C) 2013 Jakob Borg
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -1,34 +0,0 @@
# Docker Container for Syncthing
Use the Dockerfile in this repo, or pull the `syncthing/syncthing` image
from Docker Hub.
Use the `/var/syncthing` volume to have the synchronized files available on the
host. You can add more folders and map them as you prefer.
Note that Syncthing runs as UID 1000 and GID 1000 by default. These may be
altered with the ``PUID`` and ``PGID`` environment variables.
## Example Usage
```
$ docker pull syncthing/syncthing
$ docker run -p 8384:8384 -p 22000:22000 \
-v /wherever/st-sync:/var/syncthing \
syncthing/syncthing:latest
```
Note that local device discovery will not work with the above command, resulting in poor local transfer rates if local device addresses are not manually configured.
To allow local discovery, the docker host network can be used instead:
```
$ docker pull syncthing/syncthing
$ docker run --network=host \
-v /wherever/st-sync:/var/syncthing \
syncthing/syncthing:latest
```
Be aware that syncthing alone is now in control of what interfaces and ports it
listens on. You can edit the syncthing configuration to change the defaults if
there are conflicts.

131
README.md
View File

@@ -1,113 +1,38 @@
[![Syncthing][14]][15]
syncthing
=========
---
This is `syncthing`, an open BitTorrent Sync alternative. It is
currently far from ready for mass consumption, but it is a usable proof
of concept and tech demo. The following are the project goals:
[![Latest Linux & Cross Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildLinuxCross.svg?style=flat-square&label=linux+%26+cross+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildLinuxCross&guest=1)
[![Latest Windows Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildWindows.svg?style=flat-square&label=windows+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildWindows&guest=1)
[![Latest Mac Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildMac.svg?style=flat-square&label=mac+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildMac&guest=1)
[![MPLv2 License](https://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/88/badge)](https://bestpractices.coreinfrastructure.org/projects/88)
[![Go Report Card](https://goreportcard.com/badge/github.com/syncthing/syncthing)](https://goreportcard.com/report/github.com/syncthing/syncthing)
1. Define an open, secure, language neutral protocol usable for
efficient synchronization of a file repository between an arbitrary
number of nodes. This is the [Block Exchange
Protocol](https://github.com/calmh/syncthing/blob/master/protocol/PROTOCOL.md)
(BEP).
## Goals
2. Provide the reference implementation to demonstrate the usability of
said protocol. This is the `syncthing` utility.
Syncthing is a **continuous file synchronization program**. It synchronizes
files between two or more computers. We strive to fulfill the goals below.
The goals are listed in order of importance, the most important one being
the first. This is the summary version of the goal list - for more
commentary, see the full [Goals document][13].
The two are evolving together; the protocol is not to be considered
stable until syncthing 1.0 is released, at which point it is locked down
for incompatible changes.
Syncthing should be:
Syncthing does not use the BitTorrent protocol. The reasons for this are
1) we don't know if BitTorrent Sync does either, so there's nothing to
be compatible with, 2) BitTorrent includes a lot of functionality for
making sure large swarms of selfish agents behave and somehow work
towards a common goal. Here we have a much smaller swarm of cooperative
agents and a simpler approach will suffice.
1. **Safe From Data Loss**
Documentation
=============
Protecting the user's data is paramount. We take every reasonable
precaution to avoid corrupting the user's files.
The syncthing documentation is kept on the
[GitHub Wiki](https://github.com/calmh/syncthing/wiki).
2. **Secure Against Attackers**
License
=======
Again, protecting the user's data is paramount. Regardless of our other
goals we must never allow the user's data to be susceptible to
eavesdropping or modification by unauthorized parties.
3. **Easy to Use**
Syncthing should be approachable, understandable and inclusive.
4. **Automatic**
User interaction should be required only when absolutely necessary.
5. **Universally Available**
Syncthing should run on every common computer. We are mindful that the
latest technology is not always available to any given individual.
6. **For Individuals**
Syncthing is primarily about empowering the individual user with safe,
secure and easy to use file synchronization.
7. **Everything Else**
There are many things we care about that don't make it on to the list. It
is fine to optimize for these values, as long as they are not in conflict
with the stated goals above.
## Getting Started
Take a look at the [getting started guide][2].
There are a few examples for keeping Syncthing running in the background
on your system in [the etc directory][3]. There are also several [GUI
implementations][11] for Windows, Mac and Linux.
## Vote on features/bugs
We'd like to encourage you to [vote][12] on issues that matter to you.
This helps the team understand what are the biggest pain points for our users, and could potentially influence what is being worked on next.
## Getting in Touch
The first and best point of contact is the [Forum][8]. There is also an IRC
channel, `#syncthing` on [freenode][4] (with a [web client][9]), for talking
directly to developers and users. If you've found something that is clearly a
bug, feel free to report it in the [GitHub issue tracker][10].
## Building
Building Syncthing from source is easy, and there's a [guide][5]
that describes it for both Unix and Windows systems.
## Signed Releases
As of v0.10.15 and onwards release binaries are GPG signed with the key
D26E6ED000654A3E, available from https://syncthing.net/security.html and
most key servers.
There is also a built in automatic upgrade mechanism (disabled in some
distribution channels) which uses a compiled in ECDSA signature. macOS
binaries are also properly code signed.
## Documentation
Please see the [Syncthing documentation site][6].
All code is licensed under the [MPLv2 License][7].
[1]: https://docs.syncthing.net/specs/bep-v1.html
[2]: https://docs.syncthing.net/intro/getting-started.html
[3]: https://github.com/syncthing/syncthing/blob/master/etc
[4]: https://www.freenode.net/
[5]: https://docs.syncthing.net/dev/building.html
[6]: https://docs.syncthing.net/
[7]: https://github.com/syncthing/syncthing/blob/master/LICENSE
[8]: https://forum.syncthing.net/
[9]: https://kiwiirc.com/client/irc.freenode.net/#syncthing
[10]: https://github.com/syncthing/syncthing/issues
[11]: https://docs.syncthing.net/users/contrib.html#gui-wrappers
[12]: https://www.bountysource.com/teams/syncthing/issues
[13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
[14]: assets/logo-text-128.png
[15]: https://syncthing.net/
MIT

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.8 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 20 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.2 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 40 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.9 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.6 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.7 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 19 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 38 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.8 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.7 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.2 KiB

View File

File diff suppressed because one or more lines are too long

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 160 KiB

View File

File diff suppressed because one or more lines are too long

BIN
assets/st-logo.pxm Normal file
View File

Binary file not shown.

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.4 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.9 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.9 KiB

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.1 KiB

View File

Binary file not shown.

6
auto/gui.files.go Normal file
View File

File diff suppressed because one or more lines are too long

74
blocks.go Normal file
View File

@@ -0,0 +1,74 @@
package main
import (
"bytes"
"crypto/sha256"
"io"
)
type Block struct {
Offset int64
Size uint32
Hash []byte
}
// Blocks returns the blockwise hash of the reader.
func Blocks(r io.Reader, blocksize int) ([]Block, error) {
var blocks []Block
var offset int64
for {
lr := &io.LimitedReader{r, int64(blocksize)}
hf := sha256.New()
n, err := io.Copy(hf, lr)
if err != nil {
return nil, err
}
if n == 0 {
break
}
b := Block{
Offset: offset,
Size: uint32(n),
Hash: hf.Sum(nil),
}
blocks = append(blocks, b)
offset += int64(n)
}
if len(blocks) == 0 {
// Empty file
blocks = append(blocks, Block{
Offset: 0,
Size: 0,
Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55},
})
}
return blocks, nil
}
// BlockDiff returns lists of common and missing (to transform src into tgt)
// blocks. Both block lists must have been created with the same block size.
func BlockDiff(src, tgt []Block) (have, need []Block) {
if len(tgt) == 0 && len(src) != 0 {
return nil, nil
}
if len(tgt) != 0 && len(src) == 0 {
// Copy the entire file
return nil, tgt
}
for i := range tgt {
if i >= len(src) || bytes.Compare(tgt[i].Hash, src[i].Hash) != 0 {
// Copy differing block
need = append(need, tgt[i])
} else {
have = append(have, tgt[i])
}
}
return have, need
}

116
blocks_test.go Normal file
View File

@@ -0,0 +1,116 @@
package main
import (
"bytes"
"fmt"
"testing"
)
var blocksTestData = []struct {
data []byte
blocksize int
hash []string
}{
{[]byte(""), 1024, []string{
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}},
{[]byte("contents"), 1024, []string{
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8"}},
{[]byte("contents"), 9, []string{
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8"}},
{[]byte("contents"), 8, []string{
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8"}},
{[]byte("contents"), 7, []string{
"ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
"043a718774c572bd8a25adbeb1bfcd5c0256ae11cecf9f9c3f925d0e52beaf89"},
},
{[]byte("contents"), 3, []string{
"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3",
"44ad63f60af0f6db6fdde6d5186ef78176367df261fa06be3079b6c80c8adba4"},
},
{[]byte("conconts"), 3, []string{
"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
"44ad63f60af0f6db6fdde6d5186ef78176367df261fa06be3079b6c80c8adba4"},
},
{[]byte("contenten"), 3, []string{
"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3",
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3"},
},
}
func TestBlocks(t *testing.T) {
for _, test := range blocksTestData {
buf := bytes.NewBuffer(test.data)
blocks, err := Blocks(buf, test.blocksize)
if err != nil {
t.Fatal(err)
}
if l := len(blocks); l != len(test.hash) {
t.Fatalf("Incorrect number of blocks %d != %d", l, len(test.hash))
} else {
i := 0
for off := int64(0); off < int64(len(test.data)); off += int64(test.blocksize) {
if blocks[i].Offset != off {
t.Errorf("Incorrect offset for block %d: %d != %d", i, blocks[i].Offset, off)
}
bs := test.blocksize
if rem := len(test.data) - int(off); bs > rem {
bs = rem
}
if int(blocks[i].Size) != bs {
t.Errorf("Incorrect length for block %d: %d != %d", i, blocks[i].Size, bs)
}
if h := fmt.Sprintf("%x", blocks[i].Hash); h != test.hash[i] {
t.Errorf("Incorrect block hash %q != %q", h, test.hash[i])
}
i++
}
}
}
}
var diffTestData = []struct {
a string
b string
s int
d []Block
}{
{"contents", "contents", 1024, []Block{}},
{"", "", 1024, []Block{}},
{"contents", "contents", 3, []Block{}},
{"contents", "cantents", 3, []Block{{0, 3, nil}}},
{"contents", "contants", 3, []Block{{3, 3, nil}}},
{"contents", "cantants", 3, []Block{{0, 3, nil}, {3, 3, nil}}},
{"contents", "", 3, []Block{{0, 0, nil}}},
{"", "contents", 3, []Block{{0, 3, nil}, {3, 3, nil}, {6, 2, nil}}},
{"con", "contents", 3, []Block{{3, 3, nil}, {6, 2, nil}}},
{"contents", "con", 3, nil},
{"contents", "cont", 3, []Block{{3, 1, nil}}},
{"cont", "contents", 3, []Block{{3, 3, nil}, {6, 2, nil}}},
}
func TestDiff(t *testing.T) {
for i, test := range diffTestData {
a, _ := Blocks(bytes.NewBufferString(test.a), test.s)
b, _ := Blocks(bytes.NewBufferString(test.b), test.s)
_, d := BlockDiff(a, b)
if len(d) != len(test.d) {
t.Fatalf("Incorrect length for diff %d; %d != %d", i, len(d), len(test.d))
} else {
for j := range test.d {
if d[j].Offset != test.d[j].Offset {
t.Errorf("Incorrect offset for diff %d block %d; %d != %d", i, j, d[j].Offset, test.d[j].Offset)
}
if d[j].Size != test.d[j].Size {
t.Errorf("Incorrect length for diff %d block %d; %d != %d", i, j, d[j].Size, test.d[j].Size)
}
}
}
}
}

45
buffers/buffers.go Normal file
View File

@@ -0,0 +1,45 @@
package buffers
const (
largeMin = 1024
)
var (
smallBuffers = make(chan []byte, 32)
largeBuffers = make(chan []byte, 32)
)
func Get(size int) []byte {
var ch = largeBuffers
if size < largeMin {
ch = smallBuffers
}
var buf []byte
select {
case buf = <-ch:
default:
}
if len(buf) < size {
return make([]byte, size)
}
return buf[:size]
}
func Put(buf []byte) {
buf = buf[:cap(buf)]
if len(buf) == 0 {
return
}
var ch = largeBuffers
if len(buf) < largeMin {
ch = smallBuffers
}
select {
case ch <- buf:
default:
}
}

1254
build.go
View File

File diff suppressed because it is too large Load Diff

143
build.sh
View File

@@ -1,102 +1,51 @@
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'
#!/bin/bash
STTRACE=${STTRACE:-}
export COPYFILE_DISABLE=true
script() {
name="$1"
shift
go run "script/$name.go" "$@"
}
version=$(git describe --always)
buildDir=dist
build() {
go run build.go "$@"
}
if [[ $fast != yes ]] ; then
go get -d
go test ./...
fi
case "${1:-default}" in
default)
build
;;
if [[ -z $1 ]] ; then
go build -ldflags "-X main.Version $version"
elif [[ $1 == "embed" ]] ; then
embedder auto gui > auto/gui.files.go \
&& go build -ldflags "-X main.Version $version"
elif [[ $1 == "tar" ]] ; then
go build -ldflags "-X main.Version $version" \
&& mkdir syncthing-dist \
&& cp syncthing README.md LICENSE syncthing-dist \
&& tar zcvf syncthing-dist.tar.gz syncthing-dist \
&& rm -rf syncthing-dist
elif [[ $1 == "all" ]] ; then
rm -rf "$buildDir"
mkdir -p "$buildDir" || exit 1
clean)
build "$@"
;;
tar)
build "$@"
;;
assets)
build "$@"
;;
xdr)
build "$@"
;;
translate)
build "$@"
;;
deb)
build "$@"
;;
setup)
build "$@"
;;
test)
LOGGER_DISCARD=1 build test
;;
bench)
LOGGER_DISCARD=1 build bench | script benchfilter
;;
prerelease)
go run script/authors.go
build transifex
pushd man ; ./refresh.sh ; popd
git add -A gui man AUTHORS
git commit -m 'gui, man, authors: Update docs, translations, and contributors'
;;
noupgrade)
build -no-upgrade tar
;;
all)
platforms=(
darwin-amd64 dragonfly-amd64 freebsd-amd64 linux-amd64 netbsd-amd64 openbsd-amd64 solaris-amd64 windows-amd64
freebsd-386 linux-386 netbsd-386 openbsd-386 windows-386
linux-arm linux-arm64 linux-ppc64 linux-ppc64le
)
for plat in "${platforms[@]}"; do
echo Building "$plat"
goos="${plat%-*}"
goarch="${plat#*-}"
dist="tar"
if [[ $goos == "windows" ]]; then
dist="zip"
fi
build -goos "$goos" -goarch "$goarch" "$dist"
echo
done
;;
test-xunit)
(GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -v -race ./lib/... ./cmd/... || true) > tests.out
go2xunit -output tests.xml -fail < tests.out
;;
*)
echo "Unknown build command $1"
;;
esac
export GOARM=7
for os in darwin-amd64 linux-386 linux-amd64 linux-arm freebsd-386 freebsd-amd64 windows-386 windows-amd64 ; do
echo "$os"
export name="syncthing-$os"
export GOOS=${os%-*}
export GOARCH=${os#*-}
go build -ldflags "-X main.Version $version"
mkdir -p "$name"
cp README.md LICENSE "$name"
case $GOOS in
windows)
cp syncthing.exe "$buildDir/$name.exe"
mv syncthing.exe "$name"
zip -qr "$buildDir/$name.zip" "$name"
;;
*)
cp syncthing "$buildDir/$name"
mv syncthing "$name"
tar zcf "$buildDir/$name.tar.gz" "$name"
;;
esac
rm -r "$name"
done
fi

1
cmd/.gitignore vendored
View File

@@ -1 +0,0 @@
!syncthing

View File

@@ -1,143 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// This doesn't build on Windows due to the Rusage stuff.
// +build !windows
package main
import (
"flag"
"fmt"
"log"
"runtime"
"syscall"
"time"
"github.com/syncthing/syncthing/lib/rc"
)
var homeDir = "h1"
var syncthingBin = "./bin/syncthing"
var test = "scan"
func main() {
flag.StringVar(&homeDir, "home", homeDir, "Home directory location")
flag.StringVar(&syncthingBin, "bin", syncthingBin, "Binary location")
flag.StringVar(&test, "test", test, "Test to run")
flag.Parse()
switch test {
case "scan":
// scan measures the resource usage required to perform the initial
// scan, without cleaning away the database first.
testScan()
}
}
// testScan starts a process and reports on the resource usage required to
// perform the initial scan.
func testScan() {
log.Println("Starting...")
p := rc.NewProcess("127.0.0.1:8081")
if err := p.Start(syncthingBin, "-home", homeDir, "-no-browser"); err != nil {
log.Println(err)
return
}
defer p.Stop()
wallTime := awaitScanComplete(p)
report(p, wallTime)
}
// awaitScanComplete waits for a folder to transition idle->scanning and
// then scanning->idle and returns the time taken for the scan.
func awaitScanComplete(p *rc.Process) time.Duration {
log.Println("Awaiting scan completion...")
var t0, t1 time.Time
lastEvent := 0
loop:
for {
evs, err := p.Events(lastEvent)
if err != nil {
continue
}
for _, ev := range evs {
if ev.Type == "StateChanged" {
data := ev.Data.(map[string]interface{})
log.Println(ev)
if data["to"].(string) == "scanning" {
t0 = ev.Time
continue
}
if !t0.IsZero() && data["to"].(string) == "idle" {
t1 = ev.Time
break loop
}
}
lastEvent = ev.ID
}
time.Sleep(250 * time.Millisecond)
}
return t1.Sub(t0)
}
// report stops the given process and reports on its resource usage in two
// ways: human readable to stderr, and CSV to stdout.
func report(p *rc.Process, wallTime time.Duration) {
sv, err := p.SystemVersion()
if err != nil {
log.Println(err)
return
}
ss, err := p.SystemStatus()
if err != nil {
log.Println(err)
return
}
proc, err := p.Stop()
if err != nil {
return
}
rusage, ok := proc.SysUsage().(*syscall.Rusage)
if !ok {
return
}
log.Println("Version:", sv.Version)
log.Println("Alloc:", ss.Alloc/1024, "KiB")
log.Println("Sys:", ss.Sys/1024, "KiB")
log.Println("Goroutines:", ss.Goroutines)
log.Println("Wall time:", wallTime)
log.Println("Utime:", time.Duration(rusage.Utime.Nano()))
log.Println("Stime:", time.Duration(rusage.Stime.Nano()))
if runtime.GOOS == "darwin" {
// Darwin reports in bytes, Linux seems to report in KiB even
// though the manpage says otherwise.
rusage.Maxrss /= 1024
}
log.Println("MaxRSS:", rusage.Maxrss, "KiB")
fmt.Printf("%s,%d,%d,%d,%.02f,%.02f,%.02f,%d\n",
sv.Version,
ss.Alloc/1024,
ss.Sys/1024,
ss.Goroutines,
wallTime.Seconds(),
time.Duration(rusage.Utime.Nano()).Seconds(),
time.Duration(rusage.Stime.Nano()).Seconds(),
rusage.Maxrss)
}

View File

@@ -1,95 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"strings"
"github.com/syncthing/syncthing/lib/config"
)
type APIClient struct {
http.Client
cfg config.GUIConfiguration
apikey string
}
func getClient(cfg config.GUIConfiguration) *APIClient {
httpClient := http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
return net.Dial(cfg.Network(), cfg.Address())
},
},
}
return &APIClient{
Client: httpClient,
cfg: cfg,
apikey: cfg.APIKey,
}
}
func (c *APIClient) Endpoint() string {
if c.cfg.Network() == "unix" {
return "http://unix/"
}
url := c.cfg.URL()
if !strings.HasSuffix(url, "/") {
url += "/"
}
return url
}
func (c *APIClient) Do(req *http.Request) (*http.Response, error) {
req.Header.Set("X-API-Key", c.apikey)
resp, err := c.Client.Do(req)
if err != nil {
return nil, err
}
return resp, checkResponse(resp)
}
func (c *APIClient) Get(url string) (*http.Response, error) {
request, err := http.NewRequest("GET", c.Endpoint()+"rest/"+url, nil)
if err != nil {
return nil, err
}
return c.Do(request)
}
func (c *APIClient) Post(url, body string) (*http.Response, error) {
request, err := http.NewRequest("POST", c.Endpoint()+"rest/"+url, bytes.NewBufferString(body))
if err != nil {
return nil, err
}
return c.Do(request)
}
func checkResponse(response *http.Response) error {
if response.StatusCode == 404 {
return fmt.Errorf("Invalid endpoint or API call")
} else if response.StatusCode == 403 {
return fmt.Errorf("Invalid API key")
} else if response.StatusCode != 200 {
data, err := responseToBArray(response)
if err != nil {
return err
}
body := strings.TrimSpace(string(data))
return fmt.Errorf("Unexpected HTTP status returned: %s\n%s", response.Status, body)
}
return nil
}

View File

@@ -1,60 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"fmt"
"strings"
"github.com/urfave/cli"
)
var errorsCommand = cli.Command{
Name: "errors",
HideHelp: true,
Usage: "Error command group",
Subcommands: []cli.Command{
{
Name: "show",
Usage: "Show pending errors",
Action: expects(0, dumpOutput("system/error")),
},
{
Name: "push",
Usage: "Push an error to active clients",
ArgsUsage: "[error message]",
Action: expects(1, errorsPush),
},
{
Name: "clear",
Usage: "Clear pending errors",
Action: expects(0, emptyPost("system/error/clear")),
},
},
}
func errorsPush(c *cli.Context) error {
client := c.App.Metadata["client"].(*APIClient)
errStr := strings.Join(c.Args(), " ")
response, err := client.Post("system/error", strings.TrimSpace(errStr))
if err != nil {
return err
}
if response.StatusCode != 200 {
errStr = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
bytes, err := responseToBArray(response)
if err != nil {
return err
}
body := string(bytes)
if body != "" {
errStr += "\nBody: " + body
}
return fmt.Errorf(errStr)
}
return nil
}

View File

@@ -1,192 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bufio"
"crypto/tls"
"encoding/json"
"flag"
"log"
"os"
"reflect"
"strings"
"github.com/AudriusButkevicius/recli"
"github.com/flynn-archive/go-shlex"
"github.com/mattn/go-isatty"
"github.com/pkg/errors"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/locations"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/urfave/cli"
)
func main() {
// This is somewhat a hack around a chicken and egg problem.
// We need to set the home directory and potentially other flags to know where the syncthing instance is running
// in order to get it's config ... which we then use to construct the actual CLI ... at which point it's too late
// to add flags there...
homeBaseDir := locations.GetBaseDir(locations.ConfigBaseDir)
guiCfg := config.GUIConfiguration{}
flags := flag.NewFlagSet("", flag.ContinueOnError)
flags.StringVar(&guiCfg.RawAddress, "gui-address", guiCfg.RawAddress, "Override GUI address (e.g. \"http://192.0.2.42:8443\")")
flags.StringVar(&guiCfg.APIKey, "gui-apikey", guiCfg.APIKey, "Override GUI API key")
flags.StringVar(&homeBaseDir, "home", homeBaseDir, "Set configuration directory")
// Implement the same flags at the lower CLI, with the same default values (pre-parse), but do nothing with them.
// This is so that we could reuse os.Args
fakeFlags := []cli.Flag{
cli.StringFlag{
Name: "gui-address",
Value: guiCfg.RawAddress,
Usage: "Override GUI address (e.g. \"http://192.0.2.42:8443\")",
},
cli.StringFlag{
Name: "gui-apikey",
Value: guiCfg.APIKey,
Usage: "Override GUI API key",
},
cli.StringFlag{
Name: "home",
Value: homeBaseDir,
Usage: "Set configuration directory",
},
}
// Do not print usage of these flags, and ignore errors as this can't understand plenty of things
flags.Usage = func() {}
_ = flags.Parse(os.Args[1:])
// Now if the API key and address is not provided (we are not connecting to a remote instance),
// try to rip it out of the config.
if guiCfg.RawAddress == "" && guiCfg.APIKey == "" {
// Update the base directory
err := locations.SetBaseDir(locations.ConfigBaseDir, homeBaseDir)
if err != nil {
log.Fatal(errors.Wrap(err, "setting home"))
}
// Load the certs and get the ID
cert, err := tls.LoadX509KeyPair(
locations.Get(locations.CertFile),
locations.Get(locations.KeyFile),
)
if err != nil {
log.Fatal(errors.Wrap(err, "reading device ID"))
}
myID := protocol.NewDeviceID(cert.Certificate[0])
// Load the config
cfg, err := config.Load(locations.Get(locations.ConfigFile), myID)
if err != nil {
log.Fatalln(errors.Wrap(err, "loading config"))
}
guiCfg = cfg.GUI()
} else if guiCfg.Address() == "" || guiCfg.APIKey == "" {
log.Fatalln("Both -gui-address and -gui-apikey should be specified")
}
if guiCfg.Address() == "" {
log.Fatalln("Could not find GUI Address")
}
if guiCfg.APIKey == "" {
log.Fatalln("Could not find GUI API key")
}
client := getClient(guiCfg)
cfg, err := getConfig(client)
original := cfg.Copy()
if err != nil {
log.Fatalln(errors.Wrap(err, "getting config"))
}
// Copy the config and set the default flags
recliCfg := recli.DefaultConfig
recliCfg.IDTag.Name = "xml"
recliCfg.SkipTag.Name = "json"
commands, err := recli.New(recliCfg).Construct(&cfg)
if err != nil {
log.Fatalln(errors.Wrap(err, "config reflect"))
}
// Construct the actual CLI
app := cli.NewApp()
app.Name = "stcli"
app.HelpName = app.Name
app.Author = "The Syncthing Authors"
app.Usage = "Syncthing command line interface"
app.Version = strings.Replace(build.LongVersion, "syncthing", app.Name, 1)
app.Flags = fakeFlags
app.Metadata = map[string]interface{}{
"client": client,
}
app.Commands = []cli.Command{
{
Name: "config",
HideHelp: true,
Usage: "Configuration modification command group",
Subcommands: commands,
},
showCommand,
operationCommand,
errorsCommand,
}
tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
if !tty {
// Not a TTY, consume from stdin
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
input, err := shlex.Split(scanner.Text())
if err != nil {
log.Fatalln(errors.Wrap(err, "parsing input"))
}
if len(input) == 0 {
continue
}
err = app.Run(append(os.Args, input...))
if err != nil {
log.Fatalln(err)
}
}
err = scanner.Err()
if err != nil {
log.Fatalln(err)
}
} else {
err = app.Run(os.Args)
if err != nil {
log.Fatalln(err)
}
}
if !reflect.DeepEqual(cfg, original) {
body, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
log.Fatalln(err)
}
resp, err := client.Post("system/config", string(body))
if err != nil {
log.Fatalln(err)
}
if resp.StatusCode != 200 {
body, err := responseToBArray(resp)
if err != nil {
log.Fatalln(err)
}
log.Fatalln(string(body))
}
}
}

View File

@@ -1,78 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"fmt"
"github.com/urfave/cli"
)
var operationCommand = cli.Command{
Name: "operations",
HideHelp: true,
Usage: "Operation command group",
Subcommands: []cli.Command{
{
Name: "restart",
Usage: "Restart syncthing",
Action: expects(0, emptyPost("system/restart")),
},
{
Name: "shutdown",
Usage: "Shutdown syncthing",
Action: expects(0, emptyPost("system/shutdown")),
},
{
Name: "reset",
Usage: "Reset syncthing deleting all folders and devices",
Action: expects(0, emptyPost("system/reset")),
},
{
Name: "upgrade",
Usage: "Upgrade syncthing (if a newer version is available)",
Action: expects(0, emptyPost("system/upgrade")),
},
{
Name: "folder-override",
Usage: "Override changes on folder (remote for sendonly, local for receiveonly)",
ArgsUsage: "[folder id]",
Action: expects(1, foldersOverride),
},
},
}
func foldersOverride(c *cli.Context) error {
client := c.App.Metadata["client"].(*APIClient)
cfg, err := getConfig(client)
if err != nil {
return err
}
rid := c.Args()[0]
for _, folder := range cfg.Folders {
if folder.ID == rid {
response, err := client.Post("db/override", "")
if err != nil {
return err
}
if response.StatusCode != 200 {
errStr := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
bytes, err := responseToBArray(response)
if err != nil {
return err
}
body := string(bytes)
if body != "" {
errStr += "\nBody: " + body
}
return fmt.Errorf(errStr)
}
return nil
}
}
return fmt.Errorf("Folder " + rid + " not found")
}

View File

@@ -1,44 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"github.com/urfave/cli"
)
var showCommand = cli.Command{
Name: "show",
HideHelp: true,
Usage: "Show command group",
Subcommands: []cli.Command{
{
Name: "version",
Usage: "Show syncthing client version",
Action: expects(0, dumpOutput("system/version")),
},
{
Name: "config-status",
Usage: "Show configuration status, whether or not a restart is required for changes to take effect",
Action: expects(0, dumpOutput("system/config/insync")),
},
{
Name: "system",
Usage: "Show system status",
Action: expects(0, dumpOutput("system/status")),
},
{
Name: "connections",
Usage: "Report about connections to other devices",
Action: expects(0, dumpOutput("system/connections")),
},
{
Name: "usage",
Usage: "Show usage report",
Action: expects(0, dumpOutput("svc/report")),
},
},
}

View File

@@ -1,101 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"os"
"text/tabwriter"
"github.com/syncthing/syncthing/lib/config"
"github.com/urfave/cli"
)
func responseToBArray(response *http.Response) ([]byte, error) {
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, err
}
return bytes, response.Body.Close()
}
func emptyPost(url string) cli.ActionFunc {
return func(c *cli.Context) error {
client := c.App.Metadata["client"].(*APIClient)
_, err := client.Post(url, "")
return err
}
}
func dumpOutput(url string) cli.ActionFunc {
return func(c *cli.Context) error {
client := c.App.Metadata["client"].(*APIClient)
response, err := client.Get(url)
if err != nil {
return err
}
return prettyPrintResponse(c, response)
}
}
func newTableWriter() *tabwriter.Writer {
writer := new(tabwriter.Writer)
writer.Init(os.Stdout, 0, 8, 0, '\t', 0)
return writer
}
func getConfig(c *APIClient) (config.Configuration, error) {
cfg := config.Configuration{}
response, err := c.Get("system/config")
if err != nil {
return cfg, err
}
bytes, err := responseToBArray(response)
if err != nil {
return cfg, err
}
err = json.Unmarshal(bytes, &cfg)
if err == nil {
return cfg, err
}
return cfg, nil
}
func expects(n int, actionFunc cli.ActionFunc) cli.ActionFunc {
return func(ctx *cli.Context) error {
if ctx.NArg() != n {
plural := ""
if n != 1 {
plural = "s"
}
return fmt.Errorf("expected %d argument%s, got %d", n, plural, ctx.NArg())
}
return actionFunc(ctx)
}
}
func prettyPrintJSON(data interface{}) error {
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
return enc.Encode(data)
}
func prettyPrintResponse(c *cli.Context, response *http.Response) error {
bytes, err := responseToBArray(response)
if err != nil {
return err
}
var data interface{}
if err := json.Unmarshal(bytes, &data); err != nil {
return err
}
// TODO: Check flag for pretty print format
return prettyPrintJSON(data)
}

View File

@@ -1,166 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"crypto/md5"
"errors"
"flag"
"fmt"
"io"
"log"
"os"
"path/filepath"
)
func main() {
flag.Parse()
log.Println(compareDirectories(flag.Args()...))
}
// Compare a number of directories. Returns nil if the contents are identical,
// otherwise an error describing the first found difference.
func compareDirectories(dirs ...string) error {
chans := make([]chan fileInfo, len(dirs))
for i := range chans {
chans[i] = make(chan fileInfo)
}
errcs := make([]chan error, len(dirs))
abort := make(chan struct{})
for i := range dirs {
errcs[i] = startWalker(dirs[i], chans[i], abort)
}
res := make([]fileInfo, len(dirs))
for {
numDone := 0
for i := range chans {
fi, ok := <-chans[i]
if !ok {
err, hasError := <-errcs[i]
if hasError {
close(abort)
return err
}
numDone++
}
res[i] = fi
}
for i := 1; i < len(res); i++ {
if res[i] != res[0] {
close(abort)
if res[i].name < res[0].name {
return fmt.Errorf("%s missing %v (present in %s)", dirs[0], res[i], dirs[i])
} else if res[i].name > res[0].name {
return fmt.Errorf("%s missing %v (present in %s)", dirs[i], res[0], dirs[0])
}
return fmt.Errorf("Mismatch; %v (%s) != %v (%s)", res[i], dirs[i], res[0], dirs[0])
}
}
if numDone == len(dirs) {
return nil
}
}
}
type fileInfo struct {
name string
mode os.FileMode
mod int64
hash [16]byte
}
func (f fileInfo) String() string {
return fmt.Sprintf("%s %04o %d %x", f.name, f.mode, f.mod, f.hash)
}
func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan error {
walker := func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
rn, _ := filepath.Rel(dir, path)
if rn == "." {
return nil
}
if rn == ".stversions" || rn == ".stfolder" {
return filepath.SkipDir
}
var f fileInfo
if info.Mode()&os.ModeSymlink != 0 {
f = fileInfo{
name: rn,
mode: os.ModeSymlink,
}
tgt, err := os.Readlink(path)
if err != nil {
return err
}
h := md5.New()
h.Write([]byte(tgt))
hash := h.Sum(nil)
copy(f.hash[:], hash)
} else if info.IsDir() {
f = fileInfo{
name: rn,
mode: info.Mode(),
// hash and modtime zero for directories
}
} else {
f = fileInfo{
name: rn,
mode: info.Mode(),
mod: info.ModTime().Unix(),
}
sum, err := md5file(path)
if err != nil {
return err
}
f.hash = sum
}
select {
case res <- f:
return nil
case <-abort:
return errors.New("abort")
}
}
errc := make(chan error)
go func() {
err := filepath.Walk(dir, walker)
close(res)
if err != nil {
errc <- err
}
close(errc)
}()
return errc
}
func md5file(fname string) (hash [16]byte, err error) {
f, err := os.Open(fname)
if err != nil {
return
}
defer f.Close()
h := md5.New()
io.Copy(h, f)
hb := h.Sum(nil)
copy(hash[:], hb)
return
}

View File

File diff suppressed because it is too large Load Diff

View File

@@ -1,204 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"errors"
"io/ioutil"
"regexp"
"strings"
"sync"
raven "github.com/getsentry/raven-go"
"github.com/maruel/panicparse/stack"
)
const reportServer = "https://crash.syncthing.net/report/"
var loader = newGithubSourceCodeLoader()
func init() {
raven.SetSourceCodeLoader(loader)
}
var (
clients = make(map[string]*raven.Client)
clientsMut sync.Mutex
)
func sendReport(dsn, path string, report []byte) error {
pkt, err := parseReport(path, report)
if err != nil {
return err
}
clientsMut.Lock()
defer clientsMut.Unlock()
cli, ok := clients[dsn]
if !ok {
cli, err = raven.New(dsn)
if err != nil {
return err
}
clients[dsn] = cli
}
// The client sets release and such on the packet before sending, in the
// misguided idea that it knows this better than than the packet we give
// it. So we copy the values from the packet to the client first...
cli.SetRelease(pkt.Release)
cli.SetEnvironment(pkt.Environment)
defer cli.Wait()
_, errC := cli.Capture(pkt, nil)
return <-errC
}
func parseReport(path string, report []byte) (*raven.Packet, error) {
parts := bytes.SplitN(report, []byte("\n"), 2)
if len(parts) != 2 {
return nil, errors.New("no first line")
}
version, err := parseVersion(string(parts[0]))
if err != nil {
return nil, err
}
report = parts[1]
foundPanic := false
var subjectLine []byte
for {
parts = bytes.SplitN(report, []byte("\n"), 2)
if len(parts) != 2 {
return nil, errors.New("no panic line found")
}
line := parts[0]
report = parts[1]
if foundPanic {
// The previous line was our "Panic at ..." header. We are now
// at the beginning of the real panic trace and this is our
// subject line.
subjectLine = line
break
} else if bytes.HasPrefix(line, []byte("Panic at")) {
foundPanic = true
}
}
r := bytes.NewReader(report)
ctx, err := stack.ParseDump(r, ioutil.Discard, false)
if err != nil {
return nil, err
}
// Lock the source code loader to the version we are processing here.
if version.commit != "" {
// We have a commit hash, so we know exactly which source to use
loader.LockWithVersion(version.commit)
} else if strings.HasPrefix(version.tag, "v") {
// Lets hope the tag is close enough
loader.LockWithVersion(version.tag)
} else {
// Last resort
loader.LockWithVersion("master")
}
defer loader.Unlock()
var trace raven.Stacktrace
for _, gr := range ctx.Goroutines {
if gr.First {
trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
for i, sc := range gr.Stack.Calls {
trace.Frames[len(trace.Frames)-1-i] = raven.NewStacktraceFrame(0, sc.Func.Name(), sc.SrcPath, sc.Line, 3, nil)
}
break
}
}
pkt := &raven.Packet{
Message: string(subjectLine),
Platform: "go",
Release: version.tag,
Environment: version.environment(),
Tags: raven.Tags{
raven.Tag{Key: "version", Value: version.version},
raven.Tag{Key: "tag", Value: version.tag},
raven.Tag{Key: "codename", Value: version.codename},
raven.Tag{Key: "runtime", Value: version.runtime},
raven.Tag{Key: "goos", Value: version.goos},
raven.Tag{Key: "goarch", Value: version.goarch},
raven.Tag{Key: "builder", Value: version.builder},
},
Extra: raven.Extra{
"url": reportServer + path,
},
Interfaces: []raven.Interface{&trace},
}
if version.commit != "" {
pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit})
}
return pkt, nil
}
// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC
var longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)`)
type version struct {
version string // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
tag string // "v1.1.4-rc.1"
commit string // "6aaae618", blank when absent
codename string // "Erbium Earthworm"
runtime string // "go1.12.5"
goos string // "darwin"
goarch string // "amd64"
builder string // "jb@kvin.kastelo.net"
}
func (v version) environment() string {
if v.commit != "" {
return "Development"
}
if strings.Contains(v.tag, "-rc.") {
return "Candidate"
}
if strings.Contains(v.tag, "-") {
return "Beta"
}
return "Stable"
}
func parseVersion(line string) (version, error) {
m := longVersionRE.FindStringSubmatch(line)
if len(m) == 0 {
return version{}, errors.New("unintelligeble version string")
}
v := version{
version: m[1],
codename: m[2],
runtime: m[3],
goos: m[4],
goarch: m[5],
builder: m[6],
}
parts := strings.Split(v.version, "+")
v.tag = parts[0]
if len(parts) > 1 {
fields := strings.Split(parts[1], "-")
if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
v.commit = fields[1][1:]
}
}
return v, nil
}

View File

@@ -1,64 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"fmt"
"io/ioutil"
"testing"
)
func TestParseVersion(t *testing.T) {
cases := []struct {
longVersion string
parsed version
}{
{
longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`,
parsed: version{
version: "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
tag: "v1.1.4-rc.1",
commit: "6aaae618",
codename: "Erbium Earthworm",
runtime: "go1.12.5",
goos: "darwin",
goarch: "amd64",
builder: "jb@kvin.kastelo.net",
},
},
}
for _, tc := range cases {
v, err := parseVersion(tc.longVersion)
if err != nil {
t.Error(err)
continue
}
if v != tc.parsed {
t.Error(v)
}
}
}
func TestParseReport(t *testing.T) {
bs, err := ioutil.ReadFile("_testdata/panic.log")
if err != nil {
t.Fatal(err)
}
pkt, err := parseReport("1/2/345", bs)
if err != nil {
t.Fatal(err)
}
bs, err = pkt.JSON()
if err != nil {
t.Fatal(err)
}
fmt.Printf("%s\n", bs)
}

View File

@@ -1,114 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"path/filepath"
"strings"
"sync"
"time"
)
const (
urlPrefix = "https://raw.githubusercontent.com/syncthing/syncthing/"
httpTimeout = 10 * time.Second
)
type githubSourceCodeLoader struct {
mut sync.Mutex
version string
cache map[string]map[string][][]byte // version -> file -> lines
client *http.Client
}
func newGithubSourceCodeLoader() *githubSourceCodeLoader {
return &githubSourceCodeLoader{
cache: make(map[string]map[string][][]byte),
client: &http.Client{Timeout: httpTimeout},
}
}
func (l *githubSourceCodeLoader) LockWithVersion(version string) {
l.mut.Lock()
l.version = version
if _, ok := l.cache[version]; !ok {
l.cache[version] = make(map[string][][]byte)
}
}
func (l *githubSourceCodeLoader) Unlock() {
l.mut.Unlock()
}
func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]byte, int) {
filename = filepath.ToSlash(filename)
lines, ok := l.cache[l.version][filename]
if !ok {
// Cache whatever we managed to find (or nil if nothing, so we don't try again)
defer func() {
l.cache[l.version][filename] = lines
}()
knownPrefixes := []string{"/lib/", "/cmd/"}
var idx int
for _, pref := range knownPrefixes {
idx = strings.Index(filename, pref)
if idx >= 0 {
break
}
}
if idx == -1 {
return nil, 0
}
url := urlPrefix + l.version + filename[idx:]
resp, err := l.client.Get(url)
if err != nil || resp.StatusCode != http.StatusOK {
fmt.Println("Loading source:", err.Error())
return nil, 0
}
data, err := ioutil.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil {
fmt.Println("Loading source:", err.Error())
return nil, 0
}
lines = bytes.Split(data, []byte{'\n'})
}
return getLineFromLines(lines, line, context)
}
func getLineFromLines(lines [][]byte, line, context int) ([][]byte, int) {
if lines == nil {
// cached error from ReadFile: return no lines
return nil, 0
}
line-- // stack trace lines are 1-indexed
start := line - context
var idx int
if start < 0 {
start = 0
idx = line
} else {
idx = context
}
end := line + context + 1
if line >= len(lines) {
return nil, 0
}
if end > len(lines) {
end = len(lines)
}
return lines[start:end], idx
}

View File

@@ -1,160 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Command stcrashreceiver is a trivial HTTP server that allows two things:
//
// - uploading files (crash reports) named like a SHA256 hash using a PUT request
// - checking whether such file exists using a HEAD request
//
// Typically this should be deployed behind something that manages HTTPS.
package main
import (
"bytes"
"compress/gzip"
"flag"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path"
"path/filepath"
"strings"
)
const maxRequestSize = 1 << 20 // 1 MiB
func main() {
dir := flag.String("dir", ".", "Directory to store reports in")
dsn := flag.String("dsn", "", "Sentry DSN")
listen := flag.String("listen", ":22039", "HTTP listen address")
flag.Parse()
cr := &crashReceiver{
dir: *dir,
dsn: *dsn,
}
log.SetOutput(os.Stdout)
if err := http.ListenAndServe(*listen, cr); err != nil {
log.Fatalln("HTTP serve:", err)
}
}
type crashReceiver struct {
dir string
dsn string
}
func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// The final path component should be a SHA256 hash in hex, so 64 hex
// characters. We don't care about case on the request but use lower
// case internally.
reportID := strings.ToLower(path.Base(req.URL.Path))
if len(reportID) != 64 {
http.Error(w, "Bad request", http.StatusBadRequest)
return
}
for _, c := range reportID {
if c >= 'a' && c <= 'f' {
continue
}
if c >= '0' && c <= '9' {
continue
}
http.Error(w, "Bad request", http.StatusBadRequest)
return
}
// The location of the report on disk, compressed
fullPath := filepath.Join(r.dir, r.dirFor(reportID), reportID) + ".gz"
switch req.Method {
case http.MethodGet:
r.serveGet(fullPath, w, req)
case http.MethodHead:
r.serveHead(fullPath, w, req)
case http.MethodPut:
r.servePut(reportID, fullPath, w, req)
default:
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
}
}
// serveGet responds to GET requests by serving the uncompressed report.
func (r *crashReceiver) serveGet(fullPath string, w http.ResponseWriter, _ *http.Request) {
fd, err := os.Open(fullPath)
if err != nil {
http.Error(w, "Not found", http.StatusNotFound)
return
}
defer fd.Close()
gr, err := gzip.NewReader(fd)
if err != nil {
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
_, _ = io.Copy(w, gr) // best effort
}
// serveHead responds to HEAD requests by checking if the named report
// already exists in the system.
func (r *crashReceiver) serveHead(fullPath string, w http.ResponseWriter, _ *http.Request) {
if _, err := os.Lstat(fullPath); err != nil {
http.Error(w, "Not found", http.StatusNotFound)
}
}
// servePut accepts and stores the given report.
func (r *crashReceiver) servePut(reportID, fullPath string, w http.ResponseWriter, req *http.Request) {
// Ensure the destination directory exists
if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
log.Println("Creating directory:", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
// Read at most maxRequestSize of report data.
log.Println("Receiving report", reportID)
lr := io.LimitReader(req.Body, maxRequestSize)
bs, err := ioutil.ReadAll(lr)
if err != nil {
log.Println("Reading report:", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
// Compress the report for storage
buf := new(bytes.Buffer)
gw := gzip.NewWriter(buf)
_, _ = gw.Write(bs) // can't fail
gw.Close()
// Create an output file with the compressed report
err = ioutil.WriteFile(fullPath, buf.Bytes(), 0644)
if err != nil {
log.Println("Saving report:", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
// Send the report to Sentry
if r.dsn != "" {
go func() {
// There's no need for the client to have to wait for this part.
if err := sendReport(r.dsn, reportID, bs); err != nil {
log.Println("Failed to send report:", err)
}
}()
}
}
// 01234567890abcdef... => 01/23
func (r *crashReceiver) dirFor(base string) string {
return filepath.Join(base[0:2], base[2:4])
}

View File

@@ -1,113 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"crypto/rand"
"encoding/binary"
"flag"
"log"
"strings"
"time"
"github.com/syncthing/syncthing/lib/beacon"
"github.com/syncthing/syncthing/lib/discover"
"github.com/syncthing/syncthing/lib/protocol"
)
var (
all = false // print all packets, not just first from each device/source
fake = false // send fake packets to lure out other devices faster
mc = "[ff12::8384]:21027"
bc = 21027
)
var (
// Static prefix that we use when generating fake device IDs, so that we
// can recognize them ourselves. Also makes the device ID start with
// "STPROBE-" which is humanly recognizable.
randomPrefix = []byte{148, 223, 23, 4, 148}
// Our random, fake, device ID that we use when sending announcements.
myID = randomDeviceID()
)
func main() {
flag.BoolVar(&all, "all", all, "Print all received announcements (not only first)")
flag.BoolVar(&fake, "fake", fake, "Send fake announcements")
flag.StringVar(&mc, "mc", mc, "IPv6 multicast address")
flag.IntVar(&bc, "bc", bc, "IPv4 broadcast port number")
flag.Parse()
if fake {
log.Println("My ID:", myID)
}
runbeacon(beacon.NewMulticast(mc), fake)
runbeacon(beacon.NewBroadcast(bc), fake)
select {}
}
func runbeacon(bc beacon.Interface, fake bool) {
go bc.Serve()
go recv(bc)
if fake {
go send(bc)
}
}
// receives and prints discovery announcements
func recv(bc beacon.Interface) {
seen := make(map[string]bool)
for {
data, src := bc.Recv()
if m := binary.BigEndian.Uint32(data); m != discover.Magic {
log.Printf("Incorrect magic %x in announcement from %v", m, src)
continue
}
var ann discover.Announce
ann.Unmarshal(data[4:])
if ann.ID == myID {
// This is one of our own fake packets, don't print it.
continue
}
// Print announcement details for the first packet from a given
// device ID and source address, or if -all was given.
key := ann.ID.String() + src.String()
if all || !seen[key] {
log.Printf("Announcement from %v\n", src)
log.Printf(" %v at %s\n", ann.ID, strings.Join(ann.Addresses, ", "))
seen[key] = true
}
}
}
// sends fake discovery announcements once every second
func send(bc beacon.Interface) {
ann := discover.Announce{
ID: myID,
Addresses: []string{"tcp://fake.example.com:12345"},
}
bs, _ := ann.Marshal()
for {
bc.Send(bs)
time.Sleep(time.Second)
}
}
// returns a random but recognizable device ID
func randomDeviceID() protocol.DeviceID {
var id protocol.DeviceID
copy(id[:], randomPrefix)
rand.Read(id[len(randomPrefix):])
return id
}

View File

@@ -1,10 +0,0 @@
stdiscosrv
==========
This is the global discovery server for the `syncthing` project.
Usage
-----
https://docs.syncthing.net/users/stdiscosrv.html

View File

@@ -1,411 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"encoding/pem"
"fmt"
"log"
"math/rand"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"time"
"github.com/syncthing/syncthing/lib/protocol"
)
// announcement is the format received from and sent to clients
type announcement struct {
Seen time.Time `json:"seen"`
Addresses []string `json:"addresses"`
}
type apiSrv struct {
addr string
cert tls.Certificate
db database
listener net.Listener
repl replicator // optional
useHTTP bool
mapsMut sync.Mutex
misses map[string]int32
}
type requestID int64
func (i requestID) String() string {
return fmt.Sprintf("%016x", int64(i))
}
type contextKey int
const idKey contextKey = iota
func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP bool) *apiSrv {
return &apiSrv{
addr: addr,
cert: cert,
db: db,
repl: repl,
useHTTP: useHTTP,
misses: make(map[string]int32),
}
}
func (s *apiSrv) Serve() {
if s.useHTTP {
listener, err := net.Listen("tcp", s.addr)
if err != nil {
log.Println("Listen:", err)
return
}
s.listener = listener
} else {
tlsCfg := &tls.Config{
Certificates: []tls.Certificate{s.cert},
ClientAuth: tls.RequestClientCert,
SessionTicketsDisabled: true,
MinVersion: tls.VersionTLS12,
CipherSuites: []uint16{
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
},
}
tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
if err != nil {
log.Println("Listen:", err)
return
}
s.listener = tlsListener
}
http.HandleFunc("/", s.handler)
http.HandleFunc("/ping", handlePing)
srv := &http.Server{
ReadTimeout: httpReadTimeout,
WriteTimeout: httpWriteTimeout,
MaxHeaderBytes: httpMaxHeaderBytes,
}
if err := srv.Serve(s.listener); err != nil {
log.Println("Serve:", err)
}
}
var topCtx = context.Background()
func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
t0 := time.Now()
lw := NewLoggingResponseWriter(w)
defer func() {
diff := time.Since(t0)
apiRequestsSeconds.WithLabelValues(req.Method).Observe(diff.Seconds())
apiRequestsTotal.WithLabelValues(req.Method, strconv.Itoa(lw.statusCode)).Inc()
}()
reqID := requestID(rand.Int63())
ctx := context.WithValue(topCtx, idKey, reqID)
if debug {
log.Println(reqID, req.Method, req.URL)
}
var remoteIP net.IP
if s.useHTTP {
remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
} else {
addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
if err != nil {
log.Println("remoteAddr:", err)
lw.Header().Set("Retry-After", errorRetryAfterString())
http.Error(lw, "Internal Server Error", http.StatusInternalServerError)
apiRequestsTotal.WithLabelValues("no_remote_addr").Inc()
return
}
remoteIP = addr.IP
}
switch req.Method {
case "GET":
s.handleGET(ctx, lw, req)
case "POST":
s.handlePOST(ctx, remoteIP, lw, req)
default:
http.Error(lw, "Method Not Allowed", http.StatusMethodNotAllowed)
}
}
func (s *apiSrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
reqID := ctx.Value(idKey).(requestID)
deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
if err != nil {
if debug {
log.Println(reqID, "bad device param")
}
lookupRequestsTotal.WithLabelValues("bad_request").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
key := deviceID.String()
rec, err := s.db.get(key)
if err != nil {
// some sort of internal error
lookupRequestsTotal.WithLabelValues("internal_error").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
if len(rec.Addresses) == 0 {
lookupRequestsTotal.WithLabelValues("not_found").Inc()
s.mapsMut.Lock()
misses := s.misses[key]
if misses < rec.Misses {
misses = rec.Misses + 1
} else {
misses++
}
s.misses[key] = misses
s.mapsMut.Unlock()
if misses%notFoundMissesWriteInterval == 0 {
rec.Misses = misses
rec.Missed = time.Now().UnixNano()
rec.Addresses = nil
// rec.Seen retained from get
s.db.put(key, rec)
}
w.Header().Set("Retry-After", notFoundRetryAfterString(int(misses)))
http.Error(w, "Not Found", http.StatusNotFound)
return
}
lookupRequestsTotal.WithLabelValues("success").Inc()
bs, _ := json.Marshal(announcement{
Seen: time.Unix(0, rec.Seen),
Addresses: addressStrs(rec.Addresses),
})
w.Header().Set("Content-Type", "application/json")
w.Write(bs)
}
func (s *apiSrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
reqID := ctx.Value(idKey).(requestID)
rawCert := certificateBytes(req)
if rawCert == nil {
if debug {
log.Println(reqID, "no certificates")
}
announceRequestsTotal.WithLabelValues("no_certificate").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Forbidden", http.StatusForbidden)
return
}
var ann announcement
if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
if debug {
log.Println(reqID, "decode:", err)
}
announceRequestsTotal.WithLabelValues("bad_request").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
deviceID := protocol.NewDeviceID(rawCert)
addresses := fixupAddresses(remoteIP, ann.Addresses)
if len(addresses) == 0 {
announceRequestsTotal.WithLabelValues("bad_request").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
if err := s.handleAnnounce(remoteIP, deviceID, addresses); err != nil {
announceRequestsTotal.WithLabelValues("internal_error").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
announceRequestsTotal.WithLabelValues("success").Inc()
w.Header().Set("Reannounce-After", reannounceAfterString())
w.WriteHeader(http.StatusNoContent)
}
func (s *apiSrv) Stop() {
s.listener.Close()
}
func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addresses []string) error {
key := deviceID.String()
now := time.Now()
expire := now.Add(addressExpiryTime).UnixNano()
dbAddrs := make([]DatabaseAddress, len(addresses))
for i := range addresses {
dbAddrs[i].Address = addresses[i]
dbAddrs[i].Expires = expire
}
seen := now.UnixNano()
if s.repl != nil {
s.repl.send(key, dbAddrs, seen)
}
return s.db.merge(key, dbAddrs, seen)
}
func handlePing(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(204)
}
func certificateBytes(req *http.Request) []byte {
if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
return req.TLS.PeerCertificates[0].Raw
}
if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
bs := []byte(hdr)
// The certificate is in PEM format but with spaces for newlines. We
// need to reinstate the newlines for the PEM decoder. But we need to
// leave the spaces in the BEGIN and END lines - the first and last
// space - alone.
firstSpace := bytes.Index(bs, []byte(" "))
lastSpace := bytes.LastIndex(bs, []byte(" "))
for i := firstSpace + 1; i < lastSpace; i++ {
if bs[i] == ' ' {
bs[i] = '\n'
}
}
block, _ := pem.Decode(bs)
if block == nil {
// Decoding failed
return nil
}
return block.Bytes
}
return nil
}
// fixupAddresses checks the list of addresses, removing invalid ones and
// replacing unspecified IPs with the given remote IP.
func fixupAddresses(remote net.IP, addresses []string) []string {
fixed := make([]string, 0, len(addresses))
for _, annAddr := range addresses {
uri, err := url.Parse(annAddr)
if err != nil {
continue
}
host, port, err := net.SplitHostPort(uri.Host)
if err != nil {
continue
}
ip := net.ParseIP(host)
// Some classes of IP are no-go.
if ip.IsLoopback() || ip.IsMulticast() {
continue
}
if host == "" || ip.IsUnspecified() {
// Replace the unspecified IP with the request source.
// ... unless the request source is the loopback address or
// multicast/unspecified (can't happen, really).
if remote.IsLoopback() || remote.IsMulticast() || remote.IsUnspecified() {
continue
}
// Do not use IPv6 remote address if requested scheme is ...4
// (i.e., tcp4, etc.)
if strings.HasSuffix(uri.Scheme, "4") && remote.To4() == nil {
continue
}
// Do not use IPv4 remote address if requested scheme is ...6
if strings.HasSuffix(uri.Scheme, "6") && remote.To4() != nil {
continue
}
host = remote.String()
}
uri.Host = net.JoinHostPort(host, port)
fixed = append(fixed, uri.String())
}
return fixed
}
type loggingResponseWriter struct {
http.ResponseWriter
statusCode int
}
func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
return &loggingResponseWriter{w, http.StatusOK}
}
func (lrw *loggingResponseWriter) WriteHeader(code int) {
lrw.statusCode = code
lrw.ResponseWriter.WriteHeader(code)
}
func addressStrs(dbAddrs []DatabaseAddress) []string {
res := make([]string, len(dbAddrs))
for i, a := range dbAddrs {
res[i] = a.Address
}
return res
}
func errorRetryAfterString() string {
return strconv.Itoa(errorRetryAfterSeconds + rand.Intn(errorRetryFuzzSeconds))
}
func notFoundRetryAfterString(misses int) string {
retryAfterS := notFoundRetryMinSeconds + notFoundRetryIncSeconds*misses
if retryAfterS > notFoundRetryMaxSeconds {
retryAfterS = notFoundRetryMaxSeconds
}
retryAfterS += rand.Intn(notFoundRetryFuzzSeconds)
return strconv.Itoa(retryAfterS)
}
func reannounceAfterString() string {
return strconv.Itoa(reannounceAfterSeconds + rand.Intn(reannounzeFuzzSeconds))
}

View File

@@ -1,65 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"fmt"
"net"
"testing"
)
func TestFixupAddresses(t *testing.T) {
cases := []struct {
remote net.IP
in []string
out []string
}{
{ // verbatim passthrough
in: []string{"tcp://1.2.3.4:22000"},
out: []string{"tcp://1.2.3.4:22000"},
}, { // unspecified replaced by remote
remote: net.ParseIP("1.2.3.4"),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://1.2.3.4:22000", "tcp://192.0.2.42:22000"},
}, { // unspecified not used as replacement
remote: net.ParseIP("0.0.0.0"),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // unspecified not used as replacement
remote: net.ParseIP("::"),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // localhost not used as replacement
remote: net.ParseIP("127.0.0.1"),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // localhost not used as replacement
remote: net.ParseIP("::1"),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // multicast not used as replacement
remote: net.ParseIP("224.0.0.1"),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // multicast not used as replacement
remote: net.ParseIP("ff80::42"),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // explicitly announced weirdness is also filtered
remote: net.ParseIP("192.0.2.42"),
in: []string{"tcp://:22000", "tcp://127.1.2.3:22000", "tcp://[::1]:22000", "tcp://[ff80::42]:22000"},
out: []string{"tcp://192.0.2.42:22000"},
},
}
for _, tc := range cases {
out := fixupAddresses(tc.remote, tc.in)
if fmt.Sprint(out) != fmt.Sprint(tc.out) {
t.Errorf("fixupAddresses(%v, %v) => %v, expected %v", tc.remote, tc.in, out, tc.out)
}
}
}

View File

@@ -1,354 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
//go:generate go run ../../script/protofmt.go database.proto
//go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
package main
import (
"sort"
"time"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/util"
)
type clock interface {
Now() time.Time
}
type defaultClock struct{}
func (defaultClock) Now() time.Time {
return time.Now()
}
type database interface {
put(key string, rec DatabaseRecord) error
merge(key string, addrs []DatabaseAddress, seen int64) error
get(key string) (DatabaseRecord, error)
}
type levelDBStore struct {
db *leveldb.DB
inbox chan func()
stop chan struct{}
clock clock
marshalBuf []byte
}
func newLevelDBStore(dir string) (*levelDBStore, error) {
db, err := leveldb.OpenFile(dir, levelDBOptions)
if err != nil {
return nil, err
}
return &levelDBStore{
db: db,
inbox: make(chan func(), 16),
stop: make(chan struct{}),
clock: defaultClock{},
}, nil
}
func (s *levelDBStore) put(key string, rec DatabaseRecord) error {
t0 := time.Now()
defer func() {
databaseOperationSeconds.WithLabelValues(dbOpPut).Observe(time.Since(t0).Seconds())
}()
rc := make(chan error)
s.inbox <- func() {
size := rec.Size()
if len(s.marshalBuf) < size {
s.marshalBuf = make([]byte, size)
}
n, _ := rec.MarshalTo(s.marshalBuf)
rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
}
err := <-rc
if err != nil {
databaseOperations.WithLabelValues(dbOpPut, dbResError).Inc()
} else {
databaseOperations.WithLabelValues(dbOpPut, dbResSuccess).Inc()
}
return err
}
func (s *levelDBStore) merge(key string, addrs []DatabaseAddress, seen int64) error {
t0 := time.Now()
defer func() {
databaseOperationSeconds.WithLabelValues(dbOpMerge).Observe(time.Since(t0).Seconds())
}()
rc := make(chan error)
newRec := DatabaseRecord{
Addresses: addrs,
Seen: seen,
}
s.inbox <- func() {
// grab the existing record
oldRec, err := s.get(key)
if err != nil {
// "not found" is not an error from get, so this is serious
// stuff only
rc <- err
return
}
newRec = merge(newRec, oldRec)
// We replicate s.put() functionality here ourselves instead of
// calling it because we want to serialize our get above together
// with the put in the same function.
size := newRec.Size()
if len(s.marshalBuf) < size {
s.marshalBuf = make([]byte, size)
}
n, _ := newRec.MarshalTo(s.marshalBuf)
rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
}
err := <-rc
if err != nil {
databaseOperations.WithLabelValues(dbOpMerge, dbResError).Inc()
} else {
databaseOperations.WithLabelValues(dbOpMerge, dbResSuccess).Inc()
}
return err
}
func (s *levelDBStore) get(key string) (DatabaseRecord, error) {
t0 := time.Now()
defer func() {
databaseOperationSeconds.WithLabelValues(dbOpGet).Observe(time.Since(t0).Seconds())
}()
keyBs := []byte(key)
val, err := s.db.Get(keyBs, nil)
if err == leveldb.ErrNotFound {
databaseOperations.WithLabelValues(dbOpGet, dbResNotFound).Inc()
return DatabaseRecord{}, nil
}
if err != nil {
databaseOperations.WithLabelValues(dbOpGet, dbResError).Inc()
return DatabaseRecord{}, err
}
var rec DatabaseRecord
if err := rec.Unmarshal(val); err != nil {
databaseOperations.WithLabelValues(dbOpGet, dbResUnmarshalError).Inc()
return DatabaseRecord{}, nil
}
rec.Addresses = expire(rec.Addresses, s.clock.Now().UnixNano())
databaseOperations.WithLabelValues(dbOpGet, dbResSuccess).Inc()
return rec, nil
}
func (s *levelDBStore) Serve() {
t := time.NewTimer(0)
defer t.Stop()
defer s.db.Close()
// Start the statistics serve routine. It will exit with us when
// statisticsTrigger is closed.
statisticsTrigger := make(chan struct{})
statisticsDone := make(chan struct{})
go s.statisticsServe(statisticsTrigger, statisticsDone)
loop:
for {
select {
case fn := <-s.inbox:
// Run function in serialized order.
fn()
case <-t.C:
// Trigger the statistics routine to do its thing in the
// background.
statisticsTrigger <- struct{}{}
case <-statisticsDone:
// The statistics routine is done with one iteratation, schedule
// the next.
t.Reset(databaseStatisticsInterval)
case <-s.stop:
// We're done.
close(statisticsTrigger)
break loop
}
}
// Also wait for statisticsServe to return
<-statisticsDone
}
func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
defer close(done)
for range trigger {
t0 := time.Now()
nowNanos := t0.UnixNano()
cutoff24h := t0.Add(-24 * time.Hour).UnixNano()
cutoff1w := t0.Add(-7 * 24 * time.Hour).UnixNano()
cutoff2Mon := t0.Add(-60 * 24 * time.Hour).UnixNano()
current, last24h, last1w, inactive, errors := 0, 0, 0, 0, 0
iter := s.db.NewIterator(&util.Range{}, nil)
for iter.Next() {
// Attempt to unmarshal the record and count the
// failure if there's something wrong with it.
var rec DatabaseRecord
if err := rec.Unmarshal(iter.Value()); err != nil {
errors++
continue
}
// If there are addresses that have not expired it's a current
// record, otherwise account it based on when it was last seen
// (last 24 hours or last week) or finally as inactice.
switch {
case len(expire(rec.Addresses, nowNanos)) > 0:
current++
case rec.Seen > cutoff24h:
last24h++
case rec.Seen > cutoff1w:
last1w++
case rec.Seen > cutoff2Mon:
inactive++
case rec.Missed < cutoff2Mon:
// It hasn't been seen lately and we haven't recorded
// someone asking for this device in a long time either;
// delete the record.
if err := s.db.Delete(iter.Key(), nil); err != nil {
databaseOperations.WithLabelValues(dbOpDelete, dbResError).Inc()
} else {
databaseOperations.WithLabelValues(dbOpDelete, dbResSuccess).Inc()
}
default:
inactive++
}
}
iter.Release()
databaseKeys.WithLabelValues("current").Set(float64(current))
databaseKeys.WithLabelValues("last24h").Set(float64(last24h))
databaseKeys.WithLabelValues("last1w").Set(float64(last1w))
databaseKeys.WithLabelValues("inactive").Set(float64(inactive))
databaseKeys.WithLabelValues("error").Set(float64(errors))
databaseStatisticsSeconds.Set(time.Since(t0).Seconds())
// Signal that we are done and can be scheduled again.
done <- struct{}{}
}
}
func (s *levelDBStore) Stop() {
close(s.stop)
}
// merge returns the merged result of the two database records a and b. The
// result is the union of the two address sets, with the newer expiry time
// chosen for any duplicates.
func merge(a, b DatabaseRecord) DatabaseRecord {
// Both lists must be sorted for this to work.
sort.Slice(a.Addresses, func(i, j int) bool {
return a.Addresses[i].Address < a.Addresses[j].Address
})
sort.Slice(b.Addresses, func(i, j int) bool {
return b.Addresses[i].Address < b.Addresses[j].Address
})
res := DatabaseRecord{
Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
Seen: a.Seen,
}
if b.Seen > a.Seen {
res.Seen = b.Seen
}
aIdx := 0
bIdx := 0
aAddrs := a.Addresses
bAddrs := b.Addresses
loop:
for {
switch {
case aIdx == len(aAddrs) && bIdx == len(bAddrs):
// both lists are exhausted, we are done
break loop
case aIdx == len(aAddrs):
// a is exhausted, pick from b and continue
res.Addresses = append(res.Addresses, bAddrs[bIdx])
bIdx++
continue
case bIdx == len(bAddrs):
// b is exhausted, pick from a and continue
res.Addresses = append(res.Addresses, aAddrs[aIdx])
aIdx++
continue
}
// We have values left on both sides.
aVal := aAddrs[aIdx]
bVal := bAddrs[bIdx]
switch {
case aVal.Address == bVal.Address:
// update for same address, pick newer
if aVal.Expires > bVal.Expires {
res.Addresses = append(res.Addresses, aVal)
} else {
res.Addresses = append(res.Addresses, bVal)
}
aIdx++
bIdx++
case aVal.Address < bVal.Address:
// a is smallest, pick it and continue
res.Addresses = append(res.Addresses, aVal)
aIdx++
default:
// b is smallest, pick it and continue
res.Addresses = append(res.Addresses, bVal)
bIdx++
}
}
return res
}
// expire returns the list of addresses after removing expired entries.
// Expiration happen in place, so the slice given as the parameter is
// destroyed. Internal order is not preserved.
func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
i := 0
for i < len(addrs) {
if addrs[i].Expires < now {
// This item is expired. Replace it with the last in the list
// (noop if we are at the last item).
addrs[i] = addrs[len(addrs)-1]
// Wipe the last item of the list to release references to
// strings and stuff.
addrs[len(addrs)-1] = DatabaseAddress{}
// Shorten the slice.
addrs = addrs[:len(addrs)-1]
continue
}
i++
}
return addrs
}

View File

@@ -1,836 +0,0 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: database.proto
package main
import proto "github.com/gogo/protobuf/proto"
import fmt "fmt"
import math "math"
import _ "github.com/gogo/protobuf/gogoproto"
import io "io"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
type DatabaseRecord struct {
Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
Misses int32 `protobuf:"varint,2,opt,name=misses,proto3" json:"misses,omitempty"`
Seen int64 `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
Missed int64 `protobuf:"varint,4,opt,name=missed,proto3" json:"missed,omitempty"`
}
func (m *DatabaseRecord) Reset() { *m = DatabaseRecord{} }
func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
func (*DatabaseRecord) ProtoMessage() {}
func (*DatabaseRecord) Descriptor() ([]byte, []int) {
return fileDescriptor_database_0f49e029703a04f5, []int{0}
}
func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalTo(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (dst *DatabaseRecord) XXX_Merge(src proto.Message) {
xxx_messageInfo_DatabaseRecord.Merge(dst, src)
}
func (m *DatabaseRecord) XXX_Size() int {
return m.Size()
}
func (m *DatabaseRecord) XXX_DiscardUnknown() {
xxx_messageInfo_DatabaseRecord.DiscardUnknown(m)
}
var xxx_messageInfo_DatabaseRecord proto.InternalMessageInfo
type ReplicationRecord struct {
Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses"`
Seen int64 `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
}
func (m *ReplicationRecord) Reset() { *m = ReplicationRecord{} }
func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
func (*ReplicationRecord) ProtoMessage() {}
func (*ReplicationRecord) Descriptor() ([]byte, []int) {
return fileDescriptor_database_0f49e029703a04f5, []int{1}
}
func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalTo(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (dst *ReplicationRecord) XXX_Merge(src proto.Message) {
xxx_messageInfo_ReplicationRecord.Merge(dst, src)
}
func (m *ReplicationRecord) XXX_Size() int {
return m.Size()
}
func (m *ReplicationRecord) XXX_DiscardUnknown() {
xxx_messageInfo_ReplicationRecord.DiscardUnknown(m)
}
var xxx_messageInfo_ReplicationRecord proto.InternalMessageInfo
type DatabaseAddress struct {
Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
Expires int64 `protobuf:"varint,2,opt,name=expires,proto3" json:"expires,omitempty"`
}
func (m *DatabaseAddress) Reset() { *m = DatabaseAddress{} }
func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
func (*DatabaseAddress) ProtoMessage() {}
func (*DatabaseAddress) Descriptor() ([]byte, []int) {
return fileDescriptor_database_0f49e029703a04f5, []int{2}
}
func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalTo(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (dst *DatabaseAddress) XXX_Merge(src proto.Message) {
xxx_messageInfo_DatabaseAddress.Merge(dst, src)
}
func (m *DatabaseAddress) XXX_Size() int {
return m.Size()
}
func (m *DatabaseAddress) XXX_DiscardUnknown() {
xxx_messageInfo_DatabaseAddress.DiscardUnknown(m)
}
var xxx_messageInfo_DatabaseAddress proto.InternalMessageInfo
func init() {
proto.RegisterType((*DatabaseRecord)(nil), "main.DatabaseRecord")
proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
}
func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Addresses) > 0 {
for _, msg := range m.Addresses {
dAtA[i] = 0xa
i++
i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
n, err := msg.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n
}
}
if m.Misses != 0 {
dAtA[i] = 0x10
i++
i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
}
if m.Seen != 0 {
dAtA[i] = 0x18
i++
i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
}
if m.Missed != 0 {
dAtA[i] = 0x20
i++
i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
}
return i, nil
}
func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Key) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
i += copy(dAtA[i:], m.Key)
}
if len(m.Addresses) > 0 {
for _, msg := range m.Addresses {
dAtA[i] = 0x12
i++
i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
n, err := msg.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n
}
}
if m.Seen != 0 {
dAtA[i] = 0x18
i++
i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
}
return i, nil
}
func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Address) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
i += copy(dAtA[i:], m.Address)
}
if m.Expires != 0 {
dAtA[i] = 0x10
i++
i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
}
return i, nil
}
func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
dAtA[offset] = uint8(v)
return offset + 1
}
func (m *DatabaseRecord) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
if len(m.Addresses) > 0 {
for _, e := range m.Addresses {
l = e.Size()
n += 1 + l + sovDatabase(uint64(l))
}
}
if m.Misses != 0 {
n += 1 + sovDatabase(uint64(m.Misses))
}
if m.Seen != 0 {
n += 1 + sovDatabase(uint64(m.Seen))
}
if m.Missed != 0 {
n += 1 + sovDatabase(uint64(m.Missed))
}
return n
}
func (m *ReplicationRecord) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.Key)
if l > 0 {
n += 1 + l + sovDatabase(uint64(l))
}
if len(m.Addresses) > 0 {
for _, e := range m.Addresses {
l = e.Size()
n += 1 + l + sovDatabase(uint64(l))
}
}
if m.Seen != 0 {
n += 1 + sovDatabase(uint64(m.Seen))
}
return n
}
func (m *DatabaseAddress) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.Address)
if l > 0 {
n += 1 + l + sovDatabase(uint64(l))
}
if m.Expires != 0 {
n += 1 + sovDatabase(uint64(m.Expires))
}
return n
}
func sovDatabase(x uint64) (n int) {
for {
n++
x >>= 7
if x == 0 {
break
}
}
return n
}
func sozDatabase(x uint64) (n int) {
return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: DatabaseRecord: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: DatabaseRecord: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthDatabase
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Addresses = append(m.Addresses, DatabaseAddress{})
if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Misses", wireType)
}
m.Misses = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Misses |= (int32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
}
m.Seen = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Seen |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Missed", wireType)
}
m.Missed = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Missed |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipDatabase(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthDatabase
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: ReplicationRecord: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: ReplicationRecord: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthDatabase
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Key = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthDatabase
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Addresses = append(m.Addresses, DatabaseAddress{})
if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
}
m.Seen = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Seen |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipDatabase(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthDatabase
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: DatabaseAddress: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: DatabaseAddress: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Address", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthDatabase
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Address = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Expires", wireType)
}
m.Expires = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Expires |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipDatabase(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthDatabase
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipDatabase(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDatabase
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDatabase
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if dAtA[iNdEx-1] < 0x80 {
break
}
}
return iNdEx, nil
case 1:
iNdEx += 8
return iNdEx, nil
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDatabase
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
iNdEx += length
if length < 0 {
return 0, ErrInvalidLengthDatabase
}
return iNdEx, nil
case 3:
for {
var innerWire uint64
var start int = iNdEx
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDatabase
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
innerWire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
innerWireType := int(innerWire & 0x7)
if innerWireType == 4 {
break
}
next, err := skipDatabase(dAtA[start:])
if err != nil {
return 0, err
}
iNdEx = start + next
}
return iNdEx, nil
case 4:
return iNdEx, nil
case 5:
iNdEx += 4
return iNdEx, nil
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
}
panic("unreachable")
}
var (
ErrInvalidLengthDatabase = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowDatabase = fmt.Errorf("proto: integer overflow")
)
func init() { proto.RegisterFile("database.proto", fileDescriptor_database_0f49e029703a04f5) }
var fileDescriptor_database_0f49e029703a04f5 = []byte{
// 270 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
}

View File

@@ -1,36 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
syntax = "proto3";
package main;
import "repos/protobuf/gogoproto/gogo.proto";
option (gogoproto.goproto_getters_all) = false;
option (gogoproto.goproto_unkeyed_all) = false;
option (gogoproto.goproto_unrecognized_all) = false;
option (gogoproto.goproto_sizecache_all) = false;
message DatabaseRecord {
repeated DatabaseAddress addresses = 1 [(gogoproto.nullable) = false];
int32 misses = 2; // Number of lookups* without hits
int64 seen = 3; // Unix nanos, last device announce
int64 missed = 4; // Unix nanos, last* failed lookup
}
// *) Not every lookup results in a write, so may not be completely accurate
message ReplicationRecord {
string key = 1;
repeated DatabaseAddress addresses = 2 [(gogoproto.nullable) = false];
int64 seen = 3; // Unix nanos, last device announce
}
message DatabaseAddress {
string address = 1;
int64 expires = 2; // Unix nanos
}

View File

@@ -1,211 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"fmt"
"os"
"testing"
"time"
)
func TestDatabaseGetSet(t *testing.T) {
os.RemoveAll("_database")
defer os.RemoveAll("_database")
db, err := newLevelDBStore("_database")
if err != nil {
t.Fatal(err)
}
go db.Serve()
defer db.Stop()
// Check missing record
rec, err := db.get("abcd")
if err != nil {
t.Error("not found should not be an error")
}
if len(rec.Addresses) != 0 {
t.Error("addresses should be empty")
}
if rec.Misses != 0 {
t.Error("missing should be zero")
}
// Set up a clock
now := time.Now()
tc := &testClock{now}
db.clock = tc
// Put a record
rec.Addresses = []DatabaseAddress{
{Address: "tcp://1.2.3.4:5", Expires: tc.Now().Add(time.Minute).UnixNano()},
}
if err := db.put("abcd", rec); err != nil {
t.Fatal(err)
}
// Verify it
rec, err = db.get("abcd")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 1 {
t.Log(rec.Addresses)
t.Fatal("should have one address")
}
if rec.Addresses[0].Address != "tcp://1.2.3.4:5" {
t.Log(rec.Addresses)
t.Error("incorrect address")
}
// Wind the clock one half expiry, and merge in a new address
tc.wind(30 * time.Second)
addrs := []DatabaseAddress{
{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
}
if err := db.merge("abcd", addrs, tc.Now().UnixNano()); err != nil {
t.Fatal(err)
}
// Verify it
rec, err = db.get("abcd")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 2 {
t.Log(rec.Addresses)
t.Fatal("should have two addresses")
}
if rec.Addresses[0].Address != "tcp://1.2.3.4:5" {
t.Log(rec.Addresses)
t.Error("incorrect address[0]")
}
if rec.Addresses[1].Address != "tcp://6.7.8.9:0" {
t.Log(rec.Addresses)
t.Error("incorrect address[1]")
}
// Pass the first expiry time
tc.wind(45 * time.Second)
// Verify it
rec, err = db.get("abcd")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 1 {
t.Log(rec.Addresses)
t.Fatal("should have one address")
}
if rec.Addresses[0].Address != "tcp://6.7.8.9:0" {
t.Log(rec.Addresses)
t.Error("incorrect address")
}
// Put a record with misses
rec = DatabaseRecord{Misses: 42}
if err := db.put("efgh", rec); err != nil {
t.Fatal(err)
}
// Verify it
rec, err = db.get("efgh")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 0 {
t.Log(rec.Addresses)
t.Fatal("should have no addresses")
}
if rec.Misses != 42 {
t.Log(rec.Misses)
t.Error("incorrect misses")
}
// Set an address
addrs = []DatabaseAddress{
{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
}
if err := db.merge("efgh", addrs, tc.Now().UnixNano()); err != nil {
t.Fatal(err)
}
// Verify it
rec, err = db.get("efgh")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 1 {
t.Log(rec.Addresses)
t.Fatal("should have one address")
}
if rec.Misses != 0 {
t.Log(rec.Misses)
t.Error("should have no misses")
}
}
func TestFilter(t *testing.T) {
// all cases are expired with t=10
cases := []struct {
a []DatabaseAddress
b []DatabaseAddress
}{
{
a: nil,
b: nil,
},
{
a: []DatabaseAddress{{Address: "a", Expires: 9}, {Address: "b", Expires: 9}, {Address: "c", Expires: 9}},
b: []DatabaseAddress{},
},
{
a: []DatabaseAddress{{Address: "a", Expires: 10}},
b: []DatabaseAddress{{Address: "a", Expires: 10}},
},
{
a: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
b: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
},
{
a: []DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
b: []DatabaseAddress{{Address: "d", Expires: 15}, {Address: "b", Expires: 15}}, // gets reordered
},
}
for _, tc := range cases {
res := expire(tc.a, 10)
if fmt.Sprint(res) != fmt.Sprint(tc.b) {
t.Errorf("Incorrect result %v, expected %v", res, tc.b)
}
}
}
type testClock struct {
now time.Time
}
func (t *testClock) wind(d time.Duration) {
t.now = t.now.Add(d)
}
func (t *testClock) Now() time.Time {
return t.now
}

View File

@@ -1,4 +0,0 @@
[stdiscosrv]
title=Syncthing discovery server
description=Lets syncthing clients discover each other
ports=8443/tcp

View File

@@ -1,3 +0,0 @@
# Default settings for syncthing-relaysrv (strelaysrv).
## Add Options here:
DISCOSRV_OPTS=

View File

@@ -1,25 +0,0 @@
[Unit]
Description=Syncthing Discovery Server
After=network.target
Documentation=man:stdiscosrv(1)
[Service]
WorkingDirectory=/var/lib/syncthing-discosrv
EnvironmentFile=/etc/default/syncthing-discosrv
ExecStart=/usr/bin/stdiscosrv $DISCOSRV_OPTS
# Hardening
User=syncthing-discosrv
Group=syncthing
ProtectSystem=strict
ReadWritePaths=/var/lib/syncthing-discosrv
NoNewPrivileges=true
PrivateTmp=true
PrivateDevices=true
ProtectHome=true
SystemCallArchitectures=native
MemoryDenyWriteExecute=true
[Install]
WantedBy=multi-user.target
Alias=syncthing-discosrv.service

View File

@@ -1,202 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"crypto/tls"
"flag"
"fmt"
"log"
"net"
"net/http"
"os"
"runtime"
"strconv"
"strings"
"time"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/tlsutil"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/thejerf/suture"
)
const (
addressExpiryTime = 2 * time.Hour
databaseStatisticsInterval = 5 * time.Minute
// Reannounce-After is set to reannounceAfterSeconds +
// random(reannounzeFuzzSeconds), similar for Retry-After
reannounceAfterSeconds = 3300
reannounzeFuzzSeconds = 300
errorRetryAfterSeconds = 1500
errorRetryFuzzSeconds = 300
// Retry for not found is minSeconds + failures * incSeconds +
// random(fuzz), where failures is the number of consecutive lookups
// with no answer, up to maxSeconds. The fuzz is applied after capping
// to maxSeconds.
notFoundRetryMinSeconds = 60
notFoundRetryMaxSeconds = 3540
notFoundRetryIncSeconds = 10
notFoundRetryFuzzSeconds = 60
// How often (in requests) we serialize the missed counter to database.
notFoundMissesWriteInterval = 10
httpReadTimeout = 5 * time.Second
httpWriteTimeout = 5 * time.Second
httpMaxHeaderBytes = 1 << 10
// Size of the replication outbox channel
replicationOutboxSize = 10000
)
// These options make the database a little more optimized for writes, at
// the expense of some memory usage and risk of losing writes in a (system)
// crash.
var levelDBOptions = &opt.Options{
NoSync: true,
WriteBuffer: 32 << 20, // default 4<<20
}
var (
Version string
BuildStamp string
BuildUser string
BuildHost string
BuildDate time.Time
LongVersion string
)
func init() {
stamp, _ := strconv.Atoi(BuildStamp)
BuildDate = time.Unix(int64(stamp), 0)
date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
LongVersion = fmt.Sprintf(`stdiscosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
}
var (
debug = false
)
func main() {
var listen string
var dir string
var metricsListen string
var replicationListen string
var replicationPeers string
var certFile string
var keyFile string
var useHTTP bool
log.SetOutput(os.Stdout)
log.SetFlags(0)
flag.StringVar(&certFile, "cert", "./cert.pem", "Certificate file")
flag.StringVar(&dir, "db-dir", "./discovery.db", "Database directory")
flag.BoolVar(&debug, "debug", false, "Print debug output")
flag.BoolVar(&useHTTP, "http", false, "Listen on HTTP (behind an HTTPS proxy)")
flag.StringVar(&listen, "listen", ":8443", "Listen address")
flag.StringVar(&keyFile, "key", "./key.pem", "Key file")
flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
flag.Parse()
log.Println(LongVersion)
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
log.Println("Failed to load keypair. Generating one, this might take a while...")
cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv")
if err != nil {
log.Fatalln("Failed to generate X509 key pair:", err)
}
}
devID := protocol.NewDeviceID(cert.Certificate[0])
log.Println("Server device ID is", devID)
// Parse the replication specs, if any.
var allowedReplicationPeers []protocol.DeviceID
var replicationDestinations []string
parts := strings.Split(replicationPeers, ",")
for _, part := range parts {
fields := strings.Split(part, "@")
switch len(fields) {
case 2:
// This is an id@address specification. Grab the address for the
// destination list. Try to resolve it once to catch obvious
// syntax errors here rather than having the sender service fail
// repeatedly later.
_, err := net.ResolveTCPAddr("tcp", fields[1])
if err != nil {
log.Fatalln("Resolving address:", err)
}
replicationDestinations = append(replicationDestinations, fields[1])
fallthrough // N.B.
case 1:
// The first part is always a device ID.
id, err := protocol.DeviceIDFromString(fields[0])
if err != nil {
log.Fatalln("Parsing device ID:", err)
}
allowedReplicationPeers = append(allowedReplicationPeers, id)
default:
log.Fatalln("Unrecognized replication spec:", part)
}
}
// Root of the service tree.
main := suture.New("main", suture.Spec{
PassThroughPanics: true,
})
// Start the database.
db, err := newLevelDBStore(dir)
if err != nil {
log.Fatalln("Open database:", err)
}
main.Add(db)
// Start any replication senders.
var repl replicationMultiplexer
for _, dst := range replicationDestinations {
rs := newReplicationSender(dst, cert, allowedReplicationPeers)
main.Add(rs)
repl = append(repl, rs)
}
// If we have replication configured, start the replication listener.
if len(allowedReplicationPeers) > 0 {
rl := newReplicationListener(replicationListen, cert, allowedReplicationPeers, db)
main.Add(rl)
}
// Start the main API server.
qs := newAPISrv(listen, cert, db, repl, useHTTP)
main.Add(qs)
// If we have a metrics port configured, start a metrics handler.
if metricsListen != "" {
go func() {
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
log.Fatal(http.ListenAndServe(metricsListen, mux))
}()
}
// Engage!
main.Serve()
}

View File

@@ -1,326 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"crypto/tls"
"encoding/binary"
"fmt"
io "io"
"log"
"net"
"time"
"github.com/syncthing/syncthing/lib/protocol"
)
const replicationReadTimeout = time.Minute
const replicationHeartbeatInterval = time.Second * 30
type replicator interface {
send(key string, addrs []DatabaseAddress, seen int64)
}
// a replicationSender tries to connect to the remote address and provide
// them with a feed of replication updates.
type replicationSender struct {
dst string
cert tls.Certificate // our certificate
allowedIDs []protocol.DeviceID
outbox chan ReplicationRecord
stop chan struct{}
}
func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
return &replicationSender{
dst: dst,
cert: cert,
allowedIDs: allowedIDs,
outbox: make(chan ReplicationRecord, replicationOutboxSize),
stop: make(chan struct{}),
}
}
func (s *replicationSender) Serve() {
// Sleep a little at startup. Peers often restart at the same time, and
// this avoid the service failing and entering backoff state
// unnecessarily, while also reducing the reconnect rate to something
// reasonable by default.
time.Sleep(2 * time.Second)
tlsCfg := &tls.Config{
Certificates: []tls.Certificate{s.cert},
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true,
}
// Dial the TLS connection.
conn, err := tls.Dial("tcp", s.dst, tlsCfg)
if err != nil {
log.Println("Replication connect:", err)
return
}
defer func() {
conn.SetWriteDeadline(time.Now().Add(time.Second))
conn.Close()
}()
// Get the other side device ID.
remoteID, err := deviceID(conn)
if err != nil {
log.Println("Replication connect:", err)
return
}
// Verify it's in the set of allowed device IDs.
if !deviceIDIn(remoteID, s.allowedIDs) {
log.Println("Replication connect: unexpected device ID:", remoteID)
return
}
heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
defer heartBeatTicker.Stop()
// Send records.
buf := make([]byte, 1024)
for {
select {
case <-heartBeatTicker.C:
if len(s.outbox) > 0 {
// No need to send heartbeats if there are events/prevrious
// heartbeats to send, they will keep the connection alive.
continue
}
// Empty replication message is the heartbeat:
s.outbox <- ReplicationRecord{}
case rec := <-s.outbox:
// Buffer must hold record plus four bytes for size
size := rec.Size()
if len(buf) < size+4 {
buf = make([]byte, size+4)
}
// Record comes after the four bytes size
n, err := rec.MarshalTo(buf[4:])
if err != nil {
// odd to get an error here, but we haven't sent anything
// yet so it's not fatal
replicationSendsTotal.WithLabelValues("error").Inc()
log.Println("Replication marshal:", err)
continue
}
binary.BigEndian.PutUint32(buf, uint32(n))
// Send
conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
if _, err := conn.Write(buf[:4+n]); err != nil {
replicationSendsTotal.WithLabelValues("error").Inc()
log.Println("Replication write:", err)
// Yes, we are loosing the replication event here.
return
}
replicationSendsTotal.WithLabelValues("success").Inc()
case <-s.stop:
return
}
}
}
func (s *replicationSender) Stop() {
close(s.stop)
}
func (s *replicationSender) String() string {
return fmt.Sprintf("replicationSender(%q)", s.dst)
}
func (s *replicationSender) send(key string, ps []DatabaseAddress, seen int64) {
item := ReplicationRecord{
Key: key,
Addresses: ps,
}
// The send should never block. The inbox is suitably buffered for at
// least a few seconds of stalls, which shouldn't happen in practice.
select {
case s.outbox <- item:
default:
replicationSendsTotal.WithLabelValues("drop").Inc()
}
}
// a replicationMultiplexer sends to multiple replicators
type replicationMultiplexer []replicator
func (m replicationMultiplexer) send(key string, ps []DatabaseAddress, seen int64) {
for _, s := range m {
// each send is nonblocking
s.send(key, ps, seen)
}
}
// replicationListener accepts incoming connections and reads replication
// items from them. Incoming items are applied to the KV store.
type replicationListener struct {
addr string
cert tls.Certificate
allowedIDs []protocol.DeviceID
db database
stop chan struct{}
}
func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
return &replicationListener{
addr: addr,
cert: cert,
allowedIDs: allowedIDs,
db: db,
stop: make(chan struct{}),
}
}
func (l *replicationListener) Serve() {
tlsCfg := &tls.Config{
Certificates: []tls.Certificate{l.cert},
ClientAuth: tls.RequestClientCert,
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true,
}
lst, err := tls.Listen("tcp", l.addr, tlsCfg)
if err != nil {
log.Println("Replication listen:", err)
return
}
defer lst.Close()
for {
select {
case <-l.stop:
return
default:
}
// Accept a connection
conn, err := lst.Accept()
if err != nil {
log.Println("Replication accept:", err)
return
}
// Figure out the other side device ID
remoteID, err := deviceID(conn.(*tls.Conn))
if err != nil {
log.Println("Replication accept:", err)
conn.SetWriteDeadline(time.Now().Add(time.Second))
conn.Close()
continue
}
// Verify it is in the set of allowed device IDs
if !deviceIDIn(remoteID, l.allowedIDs) {
log.Println("Replication accept: unexpected device ID:", remoteID)
conn.SetWriteDeadline(time.Now().Add(time.Second))
conn.Close()
continue
}
go l.handle(conn)
}
}
func (l *replicationListener) Stop() {
close(l.stop)
}
func (l *replicationListener) String() string {
return fmt.Sprintf("replicationListener(%q)", l.addr)
}
func (l *replicationListener) handle(conn net.Conn) {
defer func() {
conn.SetWriteDeadline(time.Now().Add(time.Second))
conn.Close()
}()
buf := make([]byte, 1024)
for {
select {
case <-l.stop:
return
default:
}
conn.SetReadDeadline(time.Now().Add(replicationReadTimeout))
// First four bytes are the size
if _, err := io.ReadFull(conn, buf[:4]); err != nil {
log.Println("Replication read size:", err)
replicationRecvsTotal.WithLabelValues("error").Inc()
return
}
// Read the rest of the record
size := int(binary.BigEndian.Uint32(buf[:4]))
if len(buf) < size {
buf = make([]byte, size)
}
if size == 0 {
// Heartbeat, ignore
continue
}
if _, err := io.ReadFull(conn, buf[:size]); err != nil {
log.Println("Replication read record:", err)
replicationRecvsTotal.WithLabelValues("error").Inc()
return
}
// Unmarshal
var rec ReplicationRecord
if err := rec.Unmarshal(buf[:size]); err != nil {
log.Println("Replication unmarshal:", err)
replicationRecvsTotal.WithLabelValues("error").Inc()
continue
}
// Store
l.db.merge(rec.Key, rec.Addresses, rec.Seen)
replicationRecvsTotal.WithLabelValues("success").Inc()
}
}
func deviceID(conn *tls.Conn) (protocol.DeviceID, error) {
// Handshake may not be complete on the server side yet, which we need
// to get the client certificate.
if !conn.ConnectionState().HandshakeComplete {
if err := conn.Handshake(); err != nil {
return protocol.DeviceID{}, err
}
}
// We expect exactly one certificate.
certs := conn.ConnectionState().PeerCertificates
if len(certs) != 1 {
return protocol.DeviceID{}, fmt.Errorf("unexpected number of certificates (%d != 1)", len(certs))
}
return protocol.NewDeviceID(certs[0].Raw), nil
}
func deviceIDIn(id protocol.DeviceID, ids []protocol.DeviceID) bool {
for _, candidate := range ids {
if id == candidate {
return true
}
}
return false
}

View File

@@ -1,4 +0,0 @@
#!/bin/bash
addgroup --system syncthing
adduser --system --home /var/lib/syncthing-discosrv --ingroup syncthing syncthing-discosrv

View File

@@ -1,123 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"os"
"github.com/prometheus/client_golang/prometheus"
)
var (
apiRequestsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "api_requests_total",
Help: "Number of API requests.",
}, []string{"type", "result"})
apiRequestsSeconds = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "api_requests_seconds",
Help: "Latency of API requests.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
}, []string{"type"})
lookupRequestsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "lookup_requests_total",
Help: "Number of lookup requests.",
}, []string{"result"})
announceRequestsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "announcement_requests_total",
Help: "Number of announcement requests.",
}, []string{"result"})
replicationSendsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "replication_sends_total",
Help: "Number of replication sends.",
}, []string{"result"})
replicationRecvsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "replication_recvs_total",
Help: "Number of replication receives.",
}, []string{"result"})
databaseKeys = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "database_keys",
Help: "Number of database keys at last count.",
}, []string{"category"})
databaseStatisticsSeconds = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "database_statistics_seconds",
Help: "Time spent running the statistics routine.",
})
databaseOperations = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "database_operations_total",
Help: "Number of database operations.",
}, []string{"operation", "result"})
databaseOperationSeconds = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "database_operation_seconds",
Help: "Latency of database operations.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
}, []string{"operation"})
)
const (
dbOpGet = "get"
dbOpPut = "put"
dbOpMerge = "merge"
dbOpDelete = "delete"
dbResSuccess = "success"
dbResNotFound = "not_found"
dbResError = "error"
dbResUnmarshalError = "unmarsh_err"
)
func init() {
prometheus.MustRegister(apiRequestsTotal, apiRequestsSeconds,
lookupRequestsTotal, announceRequestsTotal,
replicationSendsTotal, replicationRecvsTotal,
databaseKeys, databaseStatisticsSeconds,
databaseOperations, databaseOperationSeconds)
processCollectorOpts := prometheus.ProcessCollectorOpts{
Namespace: "syncthing_discovery",
PidFn: func() (int, error) {
return os.Getpid(), nil
},
}
prometheus.MustRegister(
prometheus.NewProcessCollector(processCollectorOpts),
)
}

View File

@@ -1,63 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"encoding/json"
"flag"
"fmt"
"log"
"net/http"
"os"
"time"
)
type event struct {
ID int `json:"id"`
Type string `json:"type"`
Time time.Time `json:"time"`
Data map[string]interface{} `json:"data"`
}
func main() {
log.SetOutput(os.Stdout)
log.SetFlags(0)
target := flag.String("target", "localhost:8384", "Target Syncthing instance")
apikey := flag.String("apikey", "", "Syncthing API key")
flag.Parse()
if *apikey == "" {
log.Fatal("Must give -apikey argument")
}
since := 0
for {
req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d", *target, since), nil)
if err != nil {
log.Fatal(err)
}
req.Header.Set("X-API-Key", *apikey)
res, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal(err)
}
var events []event
err = json.NewDecoder(res.Body).Decode(&events)
if err != nil {
log.Fatal(err)
}
res.Body.Close()
for _, event := range events {
bs, _ := json.MarshalIndent(event, "", " ")
log.Printf("%s", bs)
since = event.ID
}
}
}

View File

@@ -1,83 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"context"
"flag"
"log"
"os"
"path/filepath"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
)
func main() {
log.SetFlags(0)
log.SetOutput(os.Stdout)
standardBlocks := flag.Bool("s", false, "Use standard block size")
flag.Parse()
path := flag.Arg(0)
if path == "" {
log.Fatal("Need one argument: path to check")
}
log.Println("File:")
log.Println(" ", filepath.Clean(path))
log.Println()
fi, err := os.Lstat(path)
if err != nil {
log.Fatal(err)
}
log.Println("Lstat:")
log.Printf(" Size: %d bytes", fi.Size())
log.Printf(" Mode: 0%o", fi.Mode())
log.Printf(" Time: %v", fi.ModTime())
log.Printf(" %d.%09d", fi.ModTime().Unix(), fi.ModTime().Nanosecond())
log.Println()
if !fi.Mode().IsDir() && !fi.Mode().IsRegular() {
fi, err = os.Stat(path)
if err != nil {
log.Fatal(err)
}
log.Println("Stat:")
log.Printf(" Size: %d bytes", fi.Size())
log.Printf(" Mode: 0%o", fi.Mode())
log.Printf(" Time: %v", fi.ModTime())
log.Printf(" %d.%09d", fi.ModTime().Unix(), fi.ModTime().Nanosecond())
log.Println()
}
if fi.Mode().IsRegular() {
log.Println("Blocks:")
fd, err := os.Open(path)
if err != nil {
log.Fatal(err)
}
blockSize := int(fi.Size())
if *standardBlocks || blockSize < protocol.MinBlockSize {
blockSize = protocol.BlockSize(fi.Size())
}
bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
if err != nil {
log.Fatal(err)
}
for _, b := range bs {
log.Println(" ", b)
}
}
}

View File

@@ -1,107 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"crypto/tls"
"errors"
"flag"
"fmt"
"net/url"
"os"
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/discover"
"github.com/syncthing/syncthing/lib/protocol"
)
var timeout = 5 * time.Second
func main() {
var server string
flag.StringVar(&server, "server", "", "Announce server (blank for default set)")
flag.DurationVar(&timeout, "timeout", timeout, "Query timeout")
flag.Usage = usage
flag.Parse()
if flag.NArg() != 1 {
flag.Usage()
os.Exit(64)
}
id, err := protocol.DeviceIDFromString(flag.Args()[0])
if err != nil {
fmt.Println(err)
os.Exit(1)
}
if server != "" {
checkServers(id, server)
} else {
checkServers(id, config.DefaultDiscoveryServers...)
}
}
type checkResult struct {
server string
addresses []string
error
}
func checkServers(deviceID protocol.DeviceID, servers ...string) {
t0 := time.Now()
resc := make(chan checkResult)
for _, srv := range servers {
srv := srv
go func() {
res := checkServer(deviceID, srv)
res.server = srv
resc <- res
}()
}
for range servers {
res := <-resc
u, _ := url.Parse(res.server)
fmt.Printf("%s (%v):\n", u.Host, time.Since(t0))
if res.error != nil {
fmt.Println(" " + res.error.Error())
}
for _, addr := range res.addresses {
fmt.Println(" address:", addr)
}
}
}
func checkServer(deviceID protocol.DeviceID, server string) checkResult {
disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
if err != nil {
return checkResult{error: err}
}
res := make(chan checkResult, 1)
time.AfterFunc(timeout, func() {
res <- checkResult{error: errors.New("timeout")}
})
go func() {
addresses, err := disco.Lookup(deviceID)
res <- checkResult{addresses: addresses, error: err}
}()
return <-res
}
func usage() {
fmt.Printf("Usage:\n\t%s [options] <device ID>\n\nOptions:\n", os.Args[0])
flag.PrintDefaults()
}

View File

@@ -1,44 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Commmand stfindignored lists ignored files under a given folder root.
package main
import (
"flag"
"fmt"
"os"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
)
func main() {
flag.Parse()
root := flag.Arg(0)
if root == "" {
root = "."
}
vfs := fs.NewWalkFilesystem(fs.NewFilesystem(fs.FilesystemTypeBasic, root))
ign := ignore.New(vfs)
if err := ign.Load(".stignore"); err != nil {
fmt.Fprintf(os.Stderr, "Fatal: loading ignores: %v\n", err)
os.Exit(1)
}
vfs.Walk(".", func(path string, info fs.FileInfo, err error) error {
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: %s: %v\n", path, err)
return fs.SkipDir
}
if ign.Match(path).IsIgnored() {
fmt.Println(path)
}
return nil
})
}

View File

@@ -1,119 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"flag"
"fmt"
"io"
"log"
"math/rand"
"os"
"path/filepath"
"time"
)
func main() {
dir := flag.String("dir", "~/files", "Directory to generate into")
files := flag.Int("files", 1000, "Number of files to create")
maxExp := flag.Int("maxexp", 20, "Max size exponent")
src := flag.String("src", "/dev/urandom", "Source of file data")
flag.Parse()
if err := generateFiles(*dir, *files, *maxExp, *src); err != nil {
log.Println(err)
}
}
func generateFiles(dir string, files, maxexp int, srcname string) error {
fd, err := os.Open(srcname)
if err != nil {
return err
}
for i := 0; i < files; i++ {
n := randomName()
if rand.Float64() < 0.05 {
// Some files and directories are dotfiles
n = "." + n
}
p0 := filepath.Join(dir, string(n[0]), n[0:2])
err = os.MkdirAll(p0, 0755)
if err != nil {
log.Fatal(err)
}
p1 := filepath.Join(p0, n)
s := int64(1 << uint(rand.Intn(maxexp)))
a := int64(128 * 1024)
if a > s {
a = s
}
s += rand.Int63n(a)
if err := generateOneFile(fd, p1, s); err != nil {
return err
}
}
return nil
}
func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
src := io.LimitReader(&inifiteReader{fd}, s)
dst, err := os.Create(p1)
if err != nil {
return err
}
_, err = io.Copy(dst, src)
if err != nil {
return err
}
err = dst.Close()
if err != nil {
return err
}
os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
return os.Chtimes(p1, t, t)
}
func randomName() string {
var b [16]byte
readRand(b[:])
return fmt.Sprintf("%x", b[:])
}
func readRand(bs []byte) (int, error) {
var r uint32
for i := range bs {
if i%4 == 0 {
r = uint32(rand.Int63())
}
bs[i] = byte(r >> uint((i%4)*8))
}
return len(bs), nil
}
type inifiteReader struct {
rd io.ReadSeeker
}
func (i *inifiteReader) Read(bs []byte) (int, error) {
n, err := i.rd.Read(bs)
if err == io.EOF {
err = nil
i.rd.Seek(0, 0)
}
return n, err
}

View File

@@ -1,83 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"encoding/binary"
"fmt"
"log"
"time"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/protocol"
)
func dump(ldb *db.Lowlevel) {
it := ldb.NewIterator(nil, nil)
for it.Next() {
key := it.Key()
switch key[0] {
case db.KeyTypeDevice:
folder := binary.BigEndian.Uint32(key[1:])
device := binary.BigEndian.Uint32(key[1+4:])
name := nulString(key[1+4+4:])
fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
var f protocol.FileInfo
err := f.Unmarshal(it.Value())
if err != nil {
log.Fatal(err)
}
fmt.Printf(" V:%v\n", f)
case db.KeyTypeGlobal:
folder := binary.BigEndian.Uint32(key[1:])
name := nulString(key[1+4:])
var flv db.VersionList
flv.Unmarshal(it.Value())
fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
case db.KeyTypeBlock:
folder := binary.BigEndian.Uint32(key[1:])
hash := key[1+4 : 1+4+32]
name := nulString(key[1+4+32:])
fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
case db.KeyTypeDeviceStatistic:
fmt.Printf("[dstat] K:%x V:%x\n", it.Key(), it.Value())
case db.KeyTypeFolderStatistic:
fmt.Printf("[fstat] K:%x V:%x\n", it.Key(), it.Value())
case db.KeyTypeVirtualMtime:
folder := binary.BigEndian.Uint32(key[1:])
name := nulString(key[1+4:])
val := it.Value()
var real, virt time.Time
real.UnmarshalBinary(val[:len(val)/2])
virt.UnmarshalBinary(val[len(val)/2:])
fmt.Printf("[mtime] F:%d N:%q R:%v V:%v\n", folder, name, real, virt)
case db.KeyTypeFolderIdx:
key := binary.BigEndian.Uint32(it.Key()[1:])
fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
case db.KeyTypeDeviceIdx:
key := binary.BigEndian.Uint32(it.Key()[1:])
val := it.Value()
if len(val) == 0 {
fmt.Printf("[deviceidx] K:%d V:<nil>\n", key)
} else {
dev := protocol.DeviceIDFromBytes(val)
fmt.Printf("[deviceidx] K:%d V:%s\n", key, dev)
}
default:
fmt.Printf("[???]\n %x\n %x\n", it.Key(), it.Value())
}
}
}

View File

@@ -1,94 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"container/heap"
"encoding/binary"
"fmt"
"github.com/syncthing/syncthing/lib/db"
)
type SizedElement struct {
key string
size int
}
type ElementHeap []SizedElement
func (h ElementHeap) Len() int { return len(h) }
func (h ElementHeap) Less(i, j int) bool { return h[i].size > h[j].size }
func (h ElementHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *ElementHeap) Push(x interface{}) {
*h = append(*h, x.(SizedElement))
}
func (h *ElementHeap) Pop() interface{} {
old := *h
n := len(old)
x := old[n-1]
*h = old[0 : n-1]
return x
}
func dumpsize(ldb *db.Lowlevel) {
h := &ElementHeap{}
heap.Init(h)
it := ldb.NewIterator(nil, nil)
var ele SizedElement
for it.Next() {
key := it.Key()
switch key[0] {
case db.KeyTypeDevice:
folder := binary.BigEndian.Uint32(key[1:])
device := binary.BigEndian.Uint32(key[1+4:])
name := nulString(key[1+4+4:])
ele.key = fmt.Sprintf("DEVICE:%d:%d:%s", folder, device, name)
case db.KeyTypeGlobal:
folder := binary.BigEndian.Uint32(key[1:])
name := nulString(key[1+4:])
ele.key = fmt.Sprintf("GLOBAL:%d:%s", folder, name)
case db.KeyTypeBlock:
folder := binary.BigEndian.Uint32(key[1:])
hash := key[1+4 : 1+4+32]
name := nulString(key[1+4+32:])
ele.key = fmt.Sprintf("BLOCK:%d:%x:%s", folder, hash, name)
case db.KeyTypeDeviceStatistic:
ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])
case db.KeyTypeFolderStatistic:
ele.key = fmt.Sprintf("FOLDERSTATS:%s", key[1:])
case db.KeyTypeVirtualMtime:
ele.key = fmt.Sprintf("MTIME:%s", key[1:])
case db.KeyTypeFolderIdx:
id := binary.BigEndian.Uint32(key[1:])
ele.key = fmt.Sprintf("FOLDERIDX:%d", id)
case db.KeyTypeDeviceIdx:
id := binary.BigEndian.Uint32(key[1:])
ele.key = fmt.Sprintf("DEVICEIDX:%d", id)
default:
ele.key = fmt.Sprintf("UNKNOWN:%x", key)
}
ele.size = len(it.Value())
heap.Push(h, ele)
}
for h.Len() > 0 {
ele = heap.Pop(h).(SizedElement)
fmt.Println(ele.key, ele.size)
}
}

View File

@@ -1,242 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"encoding/binary"
"fmt"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/protocol"
)
type fileInfoKey struct {
folder uint32
device uint32
name string
}
type globalKey struct {
folder uint32
name string
}
type sequenceKey struct {
folder uint32
sequence uint64
}
func idxck(ldb *db.Lowlevel) (success bool) {
folders := make(map[uint32]string)
devices := make(map[uint32]string)
deviceToIDs := make(map[string]uint32)
fileInfos := make(map[fileInfoKey]protocol.FileInfo)
globals := make(map[globalKey]db.VersionList)
sequences := make(map[sequenceKey]string)
needs := make(map[globalKey]struct{})
var localDeviceKey uint32
success = true
it := ldb.NewIterator(nil, nil)
for it.Next() {
key := it.Key()
switch key[0] {
case db.KeyTypeDevice:
folder := binary.BigEndian.Uint32(key[1:])
device := binary.BigEndian.Uint32(key[1+4:])
name := nulString(key[1+4+4:])
var f protocol.FileInfo
err := f.Unmarshal(it.Value())
if err != nil {
fmt.Println("Unable to unmarshal FileInfo:", err)
success = false
continue
}
fileInfos[fileInfoKey{folder, device, name}] = f
case db.KeyTypeGlobal:
folder := binary.BigEndian.Uint32(key[1:])
name := nulString(key[1+4:])
var flv db.VersionList
if err := flv.Unmarshal(it.Value()); err != nil {
fmt.Println("Unable to unmarshal VersionList:", err)
success = false
continue
}
globals[globalKey{folder, name}] = flv
case db.KeyTypeFolderIdx:
key := binary.BigEndian.Uint32(it.Key()[1:])
folders[key] = string(it.Value())
case db.KeyTypeDeviceIdx:
key := binary.BigEndian.Uint32(it.Key()[1:])
devices[key] = string(it.Value())
deviceToIDs[string(it.Value())] = key
if bytes.Equal(it.Value(), protocol.LocalDeviceID[:]) {
localDeviceKey = key
}
case db.KeyTypeSequence:
folder := binary.BigEndian.Uint32(key[1:])
seq := binary.BigEndian.Uint64(key[5:])
val := it.Value()
sequences[sequenceKey{folder, seq}] = string(val[9:])
case db.KeyTypeNeed:
folder := binary.BigEndian.Uint32(key[1:])
name := nulString(key[1+4:])
needs[globalKey{folder, name}] = struct{}{}
}
}
if localDeviceKey == 0 {
fmt.Println("Missing key for local device in device index (bailing out)")
success = false
return
}
for fk, fi := range fileInfos {
if fk.name != fi.Name {
fmt.Printf("Mismatching FileInfo name, %q (key) != %q (actual)\n", fk.name, fi.Name)
success = false
}
folder := folders[fk.folder]
if folder == "" {
fmt.Printf("Unknown folder ID %d for FileInfo %q\n", fk.folder, fk.name)
success = false
continue
}
if devices[fk.device] == "" {
fmt.Printf("Unknown device ID %d for FileInfo %q, folder %q\n", fk.folder, fk.name, folder)
success = false
}
if fk.device == localDeviceKey {
name, ok := sequences[sequenceKey{fk.folder, uint64(fi.Sequence)}]
if !ok {
fmt.Printf("Sequence entry missing for FileInfo %q, folder %q, seq %d\n", fi.Name, folder, fi.Sequence)
success = false
continue
}
if name != fi.Name {
fmt.Printf("Sequence entry refers to wrong name, %q (seq) != %q (FileInfo), folder %q, seq %d\n", name, fi.Name, folder, fi.Sequence)
success = false
}
}
}
for gk, vl := range globals {
folder := folders[gk.folder]
if folder == "" {
fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
success = false
}
for i, fv := range vl.Versions {
dev, ok := deviceToIDs[string(fv.Device)]
if !ok {
fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, fv.Device)
success = false
}
fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
if !ok {
fmt.Printf("VersionList %q, folder %q, entry %d refers to unknown FileInfo\n", gk.name, folder, i)
success = false
}
if !fi.Version.Equal(fv.Version) {
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Version, fi.Version)
success = false
}
if fi.IsInvalid() != fv.Invalid {
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Invalid, fi.IsInvalid())
success = false
}
}
// If we need this file we should have a need entry for it. False
// positives from needsLocally for deleted files, where we might
// legitimately lack an entry if we never had it, and ignored files.
if needsLocally(vl) {
_, ok := needs[gk]
if !ok {
dev := deviceToIDs[string(vl.Versions[0].Device)]
fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
if !fi.IsDeleted() && !fi.IsIgnored() {
fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
}
}
}
}
seenSeq := make(map[fileInfoKey]uint64)
for sk, name := range sequences {
folder := folders[sk.folder]
if folder == "" {
fmt.Printf("Unknown folder ID %d for sequence entry %d, %q\n", sk.folder, sk.sequence, name)
success = false
continue
}
if prev, ok := seenSeq[fileInfoKey{folder: sk.folder, name: name}]; ok {
fmt.Printf("Duplicate sequence entry for %q, folder %q, seq %d (prev %d)\n", name, folder, sk.sequence, prev)
success = false
}
seenSeq[fileInfoKey{folder: sk.folder, name: name}] = sk.sequence
fi, ok := fileInfos[fileInfoKey{sk.folder, localDeviceKey, name}]
if !ok {
fmt.Printf("Missing FileInfo for sequence entry %d, folder %q, %q\n", sk.sequence, folder, name)
success = false
continue
}
if fi.Sequence != int64(sk.sequence) {
fmt.Printf("Sequence mismatch for %q, folder %q, %d (key) != %d (FileInfo)\n", name, folder, sk.sequence, fi.Sequence)
success = false
}
}
for nk := range needs {
folder := folders[nk.folder]
if folder == "" {
fmt.Printf("Unknown folder ID %d for need entry %q\n", nk.folder, nk.name)
success = false
continue
}
vl, ok := globals[nk]
if !ok {
fmt.Printf("Missing global for need entry %q, folder %q\n", nk.name, folder)
success = false
continue
}
if !needsLocally(vl) {
fmt.Printf("Need entry for file we don't need, %q, folder %q\n", nk.name, folder)
success = false
}
}
return
}
func needsLocally(vl db.VersionList) bool {
var lv *protocol.Vector
for _, fv := range vl.Versions {
if bytes.Equal(fv.Device, protocol.LocalDeviceID[:]) {
lv = &fv.Version
break
}
}
if lv == nil {
return true // proviosinally, it looks like we need the file
}
return !lv.GreaterEqual(vl.Versions[0].Version)
}

View File

@@ -1,49 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"flag"
"fmt"
"log"
"os"
"path/filepath"
"github.com/syncthing/syncthing/lib/db"
)
func main() {
var mode string
log.SetFlags(0)
log.SetOutput(os.Stdout)
flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize, idxck")
flag.Parse()
path := flag.Arg(0)
if path == "" {
path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
}
ldb, err := db.OpenRO(path)
if err != nil {
log.Fatal(err)
}
if mode == "dump" {
dump(ldb)
} else if mode == "dumpsize" {
dumpsize(ldb)
} else if mode == "idxck" {
if !idxck(ldb) {
os.Exit(1)
}
} else {
fmt.Println("Unknown mode")
}
}

View File

@@ -1,52 +0,0 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"log"
"os"
"path/filepath"
"runtime"
"github.com/syncthing/syncthing/lib/fs"
)
func nulString(bs []byte) string {
for i := range bs {
if bs[i] == 0 {
return string(bs[:i])
}
}
return string(bs)
}
func defaultConfigDir() string {
switch runtime.GOOS {
case "windows":
if p := os.Getenv("LocalAppData"); p != "" {
return filepath.Join(p, "Syncthing")
}
return filepath.Join(os.Getenv("AppData"), "Syncthing")
case "darwin":
dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
if err != nil {
log.Fatal(err)
}
return dir
default:
if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
return filepath.Join(xdgCfg, "syncthing")
}
dir, err := fs.ExpandTilde("~/.config/syncthing")
if err != nil {
log.Fatal(err)
}
return dir
}
}

View File

@@ -1,22 +0,0 @@
The MIT License (MIT)
Copyright (c) 2015 The Syncthing Project
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -1,24 +0,0 @@
# relaypoolsrv
This is the relay pool server for the `syncthing` project, which allows
community hosted [relaysrv](https://github.com/syncthing/relaysrv)'s to join
the public pool.
Servers that join the pool are then advertised to users of `syncthing` as
potential connection points for those who are unable to connect directly due
to NAT or firewall issues.
There is very little reason why you'd want to run this yourself, as
`relaypoolsrv` is just used for announcement and lookup of public relay
servers. If you are looking to setup a private or a public relay, please
check the documentation for
[relaysrv](https://github.com/syncthing/relaysrv), which also explains how
to join the default public pool.
See `relaypoolsrv -help` for configuration options.
##### Third-party attributions
[oschwald/geoip2-golang](https://github.com/oschwald/geoip2-golang), [oschwald/maxminddb-golang](https://github.com/oschwald/maxminddb-golang), Copyright (C) 2015 [Gregory J. Oschwald](mailto:oschwald@gmail.com).
[lib/pq](https://github.com/lib/pq)</a>, Copyright (C) 2011-2013 'pq' Contributors Portions Copyright (C) 2011 Blake Mizerany.

View File

@@ -1 +0,0 @@
gui.files.go

View File

@@ -1,10 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
//go:generate go run ../../../script/genassets.go -o gui.files.go ../gui
// Package auto contains auto generated files for web assets.
package auto

View File

@@ -1,386 +0,0 @@
<!DOCTYPE html>
<html lang="en" ng-app="syncthing" ng-controller="relayDataController">
<head>
<meta charset="utf-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
<meta name="description" content=""/>
<meta name="author" content=""/>
<title>Relay stats</title>
<link href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
<link rel="stylesheet" href="//use.fontawesome.com/releases/v5.0.13/css/all.css"/>
<style>
#map {
height: 600px;
}
.ng-cloak {
display: none;
}
table {
font-size: 11px !important;
width: 100%;
border: 1px;
}
td {
padding: 0px !important;
}
tfoot td {
font-weight: bold;
}
</style>
</head>
<body class="ng-cloak">
<div class="container">
<h1>Relay Pool Data</h1>
<div ng-if="relays === undefined" class="text-center">
<img src="//cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif" alt=""/>
<p>Please wait while we gather data</p>
</div>
<div>
<div ng-show="relays !== undefined" class="ng-hide">
<p>
Currently {{ relays.length }} relays online ({{ totals.goMaxProcs }} cores in total).
</p>
</div>
<div id="map"></div> <!-- Can't hide the map, otherwise it freaks out -->
<p>The circle size represents how much bytes the relay transferred relative to other relays</p>
</div>
<div>
<table class="table table-striped table-condensed table">
<thead>
<tr>
<th rowspan="2">Address</td>
<th rowspan="2">
<a ng-click="sortType = 'stats.numActiveSessions'; sortReverse = !sortReverse">
Sessions
<span ng-show="sortType == 'stats.numActiveSessions' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.numActiveSessions' && sortReverse" class="fas fa-caret-up"></span>
</a>
</th>
<th rowspan="2">
<a ng-click="sortType = 'stats.numConnections'; sortReverse = !sortReverse">
Connections
<span ng-show="sortType == 'stats.numConnections' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.numConnections' && sortReverse" class="fas fa-caret-up"></span>
</a>
</th>
<th rowspan="2">
<a ng-click="sortType = 'stats.bytesProxied'; sortReverse = !sortReverse">
Data relayed
<span ng-show="sortType == 'stats.bytesProxied' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.bytesProxied' && sortReverse" class="fas fa-caret-up"></span>
</a>
</th>
<th colspan="6" class="text-center">Transfer rate in the last period</th>
<th rowspan="2">
<a ng-click="sortType = 'stats.uptimeSeconds'; sortReverse = !sortReverse">
Uptime hours
<span ng-show="sortType == 'stats.uptimeSeconds' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'status.uptimeSeconds' && sortReverse" class="fas fa-caret-up"></span>
</a>
</th>
<th rowspan="2">
<a ng-click="sortType = 'stats.options[\'provided-by\'] || \'\''; sortReverse = !sortReverse">
Provided by
<span ng-show="sortType == 'stats.options[\'provided-by\'] || \'\'' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.options[\'provided-by\'] || \'\'' && sortReverse" class="fas fa-caret-up"></span>
</a>
</th>
</tr>
<tr>
<th>
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[0]'; sortReverse = !sortReverse">
10s
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[0]' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[0]' && sortReverse" class="fas fa-caret-up"></span>
</a>
</th>
<th>
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[1]'; sortReverse = !sortReverse">
1m
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[1]' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[1]' && sortReverse" class="fas fa-caret-up"></span>
</a>
</th>
<th>
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[2]'; sortReverse = !sortReverse">
5m
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[2]' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[2]' && sortReverse" class="fas fa-caret-up"></span>
</a>
</th>
<th>
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[3]'; sortReverse = !sortReverse">
15m
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[3]' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[3]' && sortReverse" class="fas fa-caret-up"></span>
</a>
</th>
<th>
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[4]'; sortReverse = !sortReverse">
30m
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[4]' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[4]' && sortReverse" class="fas fa-caret-up"></span>
</a>
</th>
<th>
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[5]'; sortReverse = !sortReverse">
60m
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[5]' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[5]' && sortReverse" class="fas fa-caret-up"></span>
</a>
</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="relay in relays | orderBy:sortType:sortReverse:sortCompare" ng-mouseover="relay.showMarker()" ng-mouseleave="relay.hideMarker()">
<td>{{ relay.address }}</td>
<td ng-if="!relay.stats" colspan="11"></td>
<td ng-if-start="relay.stats">{{ relay.stats.numActiveSessions }}</td>
<td>{{ relay.stats.numConnections }}</td>
<td>{{ relay.stats.bytesProxied | bytes }}</td>
<td>{{ relay.stats.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
<td>{{ relay.stats.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
<td>{{ relay.stats.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
<td>{{ relay.stats.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
<td>{{ relay.stats.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
<td>{{ relay.stats.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
<td ng-if="relay.stats.uptimeSeconds != undefined">{{ relay.stats.uptimeSeconds/60/60 | number:0 }}</td>
<td ng-if="relay.stats.uptimeSeconds == undefined"></td>
<td title="{{ relay.stats.options['provided-by'] || '' }}" ng-if-end>
{{ relay.stats.options['provided-by'] || '' | limitTo:50 }}
<span ng-if="(relay.stats.options['provided-by'] || '').length > 50">&hellip;
</td>
</tr>
</tbody>
<tfoot>
<tr>
<td>Totals</td>
<td>{{ totals.numActiveSessions }}</td>
<td>{{ totals.numConnections }}</td>
<td>{{ totals.bytesProxied | bytes }}</td>
<td>{{ totals.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
<td>{{ totals.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
<td>{{ totals.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
<td>{{ totals.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
<td>{{ totals.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
<td>{{ totals.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
<td>{{ totals.uptimeSeconds/60/60 | number:0 }} hours</td>
<td>{{ relays.length }} relays</td>
</tr>
</tfoor>
</table>
</div>
<hr>
<p>
This product includes GeoLite2 data created by MaxMind, available from
<a href="http://www.maxmind.com">http://www.maxmind.com</a>.
</p>
</div>
<script type="text/javascript" src="//code.jquery.com/jquery-2.1.4.min.js"></script>
<script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
<script type="text/javascript" src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
<script type="text/javascript" src="//maps.googleapis.com/maps/api/js?key=AIzaSyDk5WJ8s7ueLKb99X5DbQ-vkWtPDAKqYs0"></script>
</body>
<script>
angular.module('syncthing', [
])
.config(['$httpProvider', function($httpProvider) {
$httpProvider.defaults.timeout = 5000;
}])
.filter('bytes', function() {
return function(bytes, precision) {
if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) return '-';
if (typeof precision === 'undefined') precision = 1;
var units = ['bytes', 'kB', 'MB', 'GB', 'TB', 'PB'],
number = Math.floor(Math.log(bytes) / Math.log(1024));
var value = (bytes / Math.pow(1000, Math.floor(number)));
if (!isFinite(value)) {
value = 0;
precision = 0;
}
if (!isFinite(number)) {
units = 'bytes';
} else {
units = units[number];
}
return value.toFixed(precision) + ' ' + units;
}
})
.controller('relayDataController', ['$scope', '$rootScope', '$http', '$q', '$compile', '$timeout', function($scope, $rootScope, $http, $q, $compile, $timeout) {
$scope.totals = {
bytesProxied: 0,
goMaxProcs: 0,
kbps10s1m5m15m30m60m: [0, 0, 0, 0, 0, 0],
numActiveSessions: 0,
numConnections: 0,
numPendingSessionKeys: 0,
numProxies: 0,
uptimeSeconds: 0,
};
$scope.map = new google.maps.Map(document.getElementById('map'), {
zoom: 1,
mapTypeId: google.maps.MapTypeId.ROADMAP
});
$scope.mapBounds = new google.maps.LatLngBounds();
$scope.tooltipTemplate = $('#infoTemplate').html();
$scope.usedLocations = {};
$scope.sortType = 'stats.numActiveSessions';
$scope.sortReverse = true;
$scope.sortCompare = function(a, b) {
if (a.value == b.value) {
return 0;
}
if (a.type == "undefined" || a.type == "null") {
return -1;
}
if (b.type == "undefined" || b.type == "null") {
return 1;
}
return a.value > b.value ? 1 : -1;
}
$http.get("/endpoint").then(function(response) {
$scope.relays = response.data.relays;
angular.forEach($scope.relays, function(relay) {
relay.uri = constructURI(relay.url);
relay.address = relay.url.split('/')[2];
addMarkerToMap(relay);
if (relay.stats) {
angular.forEach($scope.totals, function(value, key) {
if (typeof $scope.totals[key] == 'number') {
$scope.totals[key] += relay.stats[key];
} else if (typeof $scope.totals[key] == 'object' && $scope.totals[key] instanceof Array) {
angular.forEach($scope.totals[key], function(value, index) {
$scope.totals[key][index] += relay.stats[key][index];
});
}
});
}
});
// After the totals were calculated, add circles.
angular.forEach($scope.relays, function(relay) {
if (relay.stats) {
addCircleToMap(relay);
}
});
$scope.map.fitBounds($scope.mapBounds);
if ($scope.relays.length == 1) {
$scope.map.setZoom(13);
}
});
function addMarkerToMap(relay) {
var loc = relay.location.latitude + "," + relay.location.longitude;
// Deal with overlapping markers
while (loc in $scope.usedLocations) {
var locParts = loc.split(',');
locParts = [parseFloat(locParts[0]), parseFloat(locParts[1])];
locParts[Math.round(Math.random())] += 0.5 * (Math.random() >= 0.5 ? 1 : -1);
loc = locParts.join(',');
}
$scope.usedLocations[loc] = true;
var locParts = loc.split(',');
relay.marker = new google.maps.Marker({
map: $scope.map,
position: new google.maps.LatLng(locParts[0], locParts[1]),
title: relay.url,
});
var scope = $rootScope.$new(true);
scope.relay = relay;
relay.marker.info = new google.maps.InfoWindow({
content: $compile($scope.tooltipTemplate)(scope)[0],
});
relay.showMarker = function() {
relay.marker.info.open($scope.map, relay.marker);
}
relay.hideMarker = function() {
relay.marker.info.close();
}
relay.marker.addListener('mouseover', relay.showMarker);
relay.marker.addListener('mouseout', relay.hideMarker);
$scope.mapBounds.extend(relay.marker.position);
}
function addCircleToMap(relay) {
relay.marker.circle = new google.maps.Circle({
strokeColor: '#FF0000',
strokeOpacity: 0.8,
strokeWeight: 2,
fillColor: '#FF0000',
fillOpacity: 0.35,
map: $scope.map,
center: relay.marker.position,
radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000
});
}
function constructURI(url) {
var uri = document.createElement('a');
// HAX, otherwise doesn't work
uri.href = url.replace('relay://', 'http://');
// Convert query string to object
uri.args = {};
angular.forEach(uri.search.replace(/^\?/, '').split('&'), function(query) {
var split = query.split('=');
uri.args[split[0]] = split[1];
});
return uri;
}
}]);
</script>
<script type="text/template" id="infoTemplate">
<div>
<p><b>{{ relay.uri.hostname }}</b> <span ng-if="relay.stats.options['provided-by']">provided by <u>{{ relay.stats.options['provided-by'] }}</u></span></p>
<div ng-if="relay.stats">
<span ng-if="relay.stats.startTime">Start time: {{ relay.stats.startTime | date:"medium" }}</br></span>
<span ng-if="relay.stats.bytesProxied != undefined">Proxied: {{ relay.stats.bytesProxied | bytes }}</br></span>
<span ng-if="relay.stats.numActiveSessions != undefined">Sessions: {{ relay.stats.numActiveSessions }}</br></span>
<span ng-if="relay.stats.numConnections != undefined">Clients: {{ relay.stats.numConnections }}</br></span>
<span ng-if="relay.stats.options.pools">Pools: {{ relay.stats.options.pools.join(', ') }}</br></span>
<span ng-if="relay.stats.options['global-rate'] != undefined">
<span ng-if="relay.stats.options['global-rate'] > 0">Global rate limit: {{ relay.stats.options['global-rate'] | bytes }}/s</span>
<span ng-if="relay.stats.options['global-rate'] == 0">Global rate limit: unlimited</span>
<br/>
</span>
<span ng-if="relay.stats.options['per-session-rate'] != undefined">
<span ng-if="relay.stats.options['per-session-rate'] > 0">Session rate limit: {{ relay.stats.options['per-session-rate'] | bytes }}/s</span>
<span ng-if="relay.stats.options['per-session-rate'] == 0">Session rate limit: unlimited</span>
<br/>
</span>
</div>
<div ng-if="!relay.stats">
Data unavailable.
<div>
</div>
</script>
</html>

View File

@@ -1,684 +0,0 @@
// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
//go:generate go run ../../script/genassets.go gui >auto/gui.go
package main
import (
"bytes"
"compress/gzip"
"crypto/tls"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log"
"mime"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/golang/groupcache/lru"
"github.com/oschwald/geoip2-golang"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/relay/client"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/tlsutil"
"golang.org/x/time/rate"
)
type location struct {
Latitude float64 `json:"latitude"`
Longitude float64 `json:"longitude"`
City string `json:"city"`
Country string `json:"country"`
Continent string `json:"continent"`
}
type relay struct {
URL string `json:"url"`
Location location `json:"location"`
uri *url.URL
Stats *stats `json:"stats"`
StatsRetrieved time.Time `json:"statsRetrieved"`
}
type stats struct {
StartTime time.Time `json:"startTime"`
UptimeSeconds int `json:"uptimeSeconds"`
PendingSessionKeys int `json:"numPendingSessionKeys"`
ActiveSessions int `json:"numActiveSessions"`
Connections int `json:"numConnections"`
Proxies int `json:"numProxies"`
BytesProxied int `json:"bytesProxied"`
GoVersion string `json:"goVersion"`
GoOS string `json:"goOS"`
GoArch string `json:"goArch"`
GoMaxProcs int `json:"goMaxProcs"`
GoRoutines int `json:"goNumRoutine"`
Rates []int64 `json:"kbps10s1m5m15m30m60m"`
Options struct {
NetworkTimeout int `json:"network-timeout"`
PintInterval int `json:"ping-interval"`
MessageTimeout int `json:"message-timeout"`
SessionRate int `json:"per-session-rate"`
GlobalRate int `json:"global-rate"`
Pools []string `json:"pools"`
ProvidedBy string `json:"provided-by"`
} `json:"options"`
}
func (r relay) String() string {
return r.URL
}
type request struct {
relay *relay
result chan result
queueTimer *prometheus.Timer
}
type result struct {
err error
eviction time.Duration
}
var (
testCert tls.Certificate
knownRelaysFile = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
listen = ":80"
dir string
evictionTime = time.Hour
debug bool
getLRUSize = 10 << 10
getLimitBurst = 10
getLimitAvg = 2
postLRUSize = 1 << 10
postLimitBurst = 2
postLimitAvg = 2
getLimit time.Duration
postLimit time.Duration
permRelaysFile string
ipHeader string
geoipPath string
proto string
statsRefresh = time.Minute / 2
getMut = sync.NewRWMutex()
getLRUCache *lru.Cache
postMut = sync.NewRWMutex()
postLRUCache *lru.Cache
requests = make(chan request, 10)
mut = sync.NewRWMutex()
knownRelays = make([]*relay, 0)
permanentRelays = make([]*relay, 0)
evictionTimers = make(map[string]*time.Timer)
)
const (
httpStatusEnhanceYourCalm = 429
)
func main() {
flag.StringVar(&listen, "listen", listen, "Listen address")
flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
flag.BoolVar(&debug, "debug", debug, "Enable debug output")
flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
flag.IntVar(&getLRUSize, "get-limit-cache", getLRUSize, "Get request limiter cache size")
flag.IntVar(&getLimitAvg, "get-limit-avg", getLimitAvg, "Allowed average get request rate, per 10 s")
flag.IntVar(&getLimitBurst, "get-limit-burst", getLimitBurst, "Allowed burst get requests")
flag.IntVar(&postLRUSize, "post-limit-cache", postLRUSize, "Post request limiter cache size")
flag.IntVar(&postLimitAvg, "post-limit-avg", postLimitAvg, "Allowed average post request rate, per minute")
flag.IntVar(&postLimitBurst, "post-limit-burst", postLimitBurst, "Allowed burst post requests")
flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
flag.Parse()
getLimit = 10 * time.Second / time.Duration(getLimitAvg)
postLimit = time.Minute / time.Duration(postLimitAvg)
getLRUCache = lru.New(getLRUSize)
postLRUCache = lru.New(postLRUSize)
var listener net.Listener
var err error
if permRelaysFile != "" {
permanentRelays = loadRelays(permRelaysFile)
}
testCert = createTestCertificate()
go requestProcessor()
// Load relays from cache in the background.
// Load them in a serial fashion to make sure any genuine requests
// are not dropped.
go func() {
for _, relay := range loadRelays(knownRelaysFile) {
resultChan := make(chan result)
requests <- request{relay, resultChan, nil}
result := <-resultChan
if result.err != nil {
relayTestsTotal.WithLabelValues("failed").Inc()
} else {
relayTestsTotal.WithLabelValues("success").Inc()
}
}
// Run the the stats refresher once the relays are loaded.
statsRefresher(statsRefresh)
}()
if dir != "" {
if debug {
log.Println("Starting TLS listener on", listen)
}
certFile, keyFile := filepath.Join(dir, "http-cert.pem"), filepath.Join(dir, "http-key.pem")
var cert tls.Certificate
cert, err = tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
log.Fatalln("Failed to load HTTP X509 key pair:", err)
}
tlsCfg := &tls.Config{
Certificates: []tls.Certificate{cert},
MinVersion: tls.VersionTLS10, // No SSLv3
CipherSuites: []uint16{
// No RC4
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
tls.TLS_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,
},
}
listener, err = tls.Listen(proto, listen, tlsCfg)
} else {
if debug {
log.Println("Starting plain listener on", listen)
}
listener, err = net.Listen(proto, listen)
}
if err != nil {
log.Fatalln("listen:", err)
}
handler := http.NewServeMux()
handler.HandleFunc("/", handleAssets)
handler.HandleFunc("/endpoint", handleRequest)
handler.HandleFunc("/metrics", handleMetrics)
srv := http.Server{
Handler: handler,
ReadTimeout: 10 * time.Second,
}
err = srv.Serve(listener)
if err != nil {
log.Fatalln("serve:", err)
}
}
func handleMetrics(w http.ResponseWriter, r *http.Request) {
timer := prometheus.NewTimer(metricsRequestsSeconds)
// Acquire the mutex just to make sure we're not caught mid-way stats collection
mut.RLock()
promhttp.Handler().ServeHTTP(w, r)
mut.RUnlock()
timer.ObserveDuration()
}
func handleAssets(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "no-cache, must-revalidate")
assets := auto.Assets()
path := r.URL.Path[1:]
if path == "" {
path = "index.html"
}
bs, ok := assets[path]
if !ok {
w.WriteHeader(http.StatusNotFound)
return
}
etag := fmt.Sprintf("%d", auto.Generated)
modified := time.Unix(auto.Generated, 0).UTC()
w.Header().Set("Last-Modified", modified.Format(http.TimeFormat))
w.Header().Set("Etag", etag)
mtype := mimeTypeForFile(path)
if len(mtype) != 0 {
w.Header().Set("Content-Type", mtype)
}
if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modified.Add(time.Second).After(t) {
w.WriteHeader(http.StatusNotModified)
return
}
if match := r.Header.Get("If-None-Match"); match != "" {
if strings.Contains(match, etag) {
w.WriteHeader(http.StatusNotModified)
return
}
}
if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
w.Header().Set("Content-Encoding", "gzip")
} else {
// ungzip if browser not send gzip accepted header
var gr *gzip.Reader
gr, _ = gzip.NewReader(bytes.NewReader(bs))
bs, _ = ioutil.ReadAll(gr)
gr.Close()
}
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs)))
w.Write(bs)
}
func mimeTypeForFile(file string) string {
// We use a built in table of the common types since the system
// TypeByExtension might be unreliable. But if we don't know, we delegate
// to the system.
ext := filepath.Ext(file)
switch ext {
case ".htm", ".html":
return "text/html"
case ".css":
return "text/css"
case ".js":
return "application/javascript"
case ".json":
return "application/json"
case ".png":
return "image/png"
case ".ttf":
return "application/x-font-ttf"
case ".woff":
return "application/x-font-woff"
case ".svg":
return "image/svg+xml"
default:
return mime.TypeByExtension(ext)
}
}
func handleRequest(w http.ResponseWriter, r *http.Request) {
timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
lw := NewLoggingResponseWriter(w)
defer func() {
timer.ObserveDuration()
apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
}()
if ipHeader != "" {
r.RemoteAddr = r.Header.Get(ipHeader)
}
w.Header().Set("Access-Control-Allow-Origin", "*")
switch r.Method {
case "GET":
if limit(r.RemoteAddr, getLRUCache, getMut, getLimit, getLimitBurst) {
w.WriteHeader(httpStatusEnhanceYourCalm)
return
}
handleGetRequest(w, r)
case "POST":
if limit(r.RemoteAddr, postLRUCache, postMut, postLimit, postLimitBurst) {
w.WriteHeader(httpStatusEnhanceYourCalm)
return
}
handlePostRequest(w, r)
default:
if debug {
log.Println("Unhandled HTTP method", r.Method)
}
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
}
}
func handleGetRequest(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
mut.RLock()
relays := append(permanentRelays, knownRelays...)
mut.RUnlock()
// Shuffle
rand.Shuffle(relays)
json.NewEncoder(w).Encode(map[string][]*relay{
"relays": relays,
})
}
func handlePostRequest(w http.ResponseWriter, r *http.Request) {
var newRelay relay
err := json.NewDecoder(r.Body).Decode(&newRelay)
r.Body.Close()
if err != nil {
if debug {
log.Println("Failed to parse payload")
}
http.Error(w, err.Error(), 500)
return
}
uri, err := url.Parse(newRelay.URL)
if err != nil {
if debug {
log.Println("Failed to parse URI", newRelay.URL)
}
http.Error(w, err.Error(), 500)
return
}
host, port, err := net.SplitHostPort(uri.Host)
if err != nil {
if debug {
log.Println("Failed to split URI", newRelay.URL)
}
http.Error(w, err.Error(), 500)
return
}
// Get the IP address of the client
rhost := r.RemoteAddr
if host, _, err := net.SplitHostPort(rhost); err == nil {
rhost = host
}
ip := net.ParseIP(host)
// The client did not provide an IP address, use the IP address of the client.
if ip == nil || ip.IsUnspecified() {
uri.Host = net.JoinHostPort(rhost, port)
newRelay.URL = uri.String()
} else if host != rhost {
if debug {
log.Println("IP address advertised does not match client IP address", r.RemoteAddr, uri)
}
http.Error(w, fmt.Sprintf("IP advertised %s does not match client IP %s", host, rhost), http.StatusUnauthorized)
return
}
newRelay.uri = uri
for _, current := range permanentRelays {
if current.uri.Host == newRelay.uri.Host {
if debug {
log.Println("Asked to add a relay", newRelay, "which exists in permanent list")
}
http.Error(w, "Invalid request", http.StatusBadRequest)
return
}
}
reschan := make(chan result)
select {
case requests <- request{&newRelay, reschan, prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("queue"))}:
result := <-reschan
if result.err != nil {
relayTestsTotal.WithLabelValues("failed").Inc()
http.Error(w, result.err.Error(), http.StatusBadRequest)
return
}
relayTestsTotal.WithLabelValues("success").Inc()
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(map[string]time.Duration{
"evictionIn": result.eviction,
})
default:
relayTestsTotal.WithLabelValues("dropped").Inc()
if debug {
log.Println("Dropping request")
}
w.WriteHeader(httpStatusEnhanceYourCalm)
}
}
func requestProcessor() {
for request := range requests {
if request.queueTimer != nil {
request.queueTimer.ObserveDuration()
}
timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test"))
handleRelayTest(request)
timer.ObserveDuration()
}
}
func handleRelayTest(request request) {
if debug {
log.Println("Request for", request.relay)
}
if !client.TestRelay(request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
if debug {
log.Println("Test for relay", request.relay, "failed")
}
request.result <- result{fmt.Errorf("connection test failed"), 0}
return
}
stats := fetchStats(request.relay)
location := getLocation(request.relay.uri.Host)
mut.Lock()
if stats != nil {
updateMetrics(request.relay.uri.Host, *stats, location)
}
request.relay.Stats = stats
request.relay.StatsRetrieved = time.Now()
request.relay.Location = location
timer, ok := evictionTimers[request.relay.uri.Host]
if ok {
if debug {
log.Println("Stopping existing timer for", request.relay)
}
timer.Stop()
}
for i, current := range knownRelays {
if current.uri.Host == request.relay.uri.Host {
if debug {
log.Println("Relay", request.relay, "already exists")
}
// Evict the old entry anyway, as configuration might have changed.
last := len(knownRelays) - 1
knownRelays[i] = knownRelays[last]
knownRelays = knownRelays[:last]
goto found
}
}
if debug {
log.Println("Adding new relay", request.relay)
}
found:
knownRelays = append(knownRelays, request.relay)
evictionTimers[request.relay.uri.Host] = time.AfterFunc(evictionTime, evict(request.relay))
mut.Unlock()
if err := saveRelays(knownRelaysFile, knownRelays); err != nil {
log.Println("Failed to write known relays: " + err.Error())
}
request.result <- result{nil, evictionTime}
}
func evict(relay *relay) func() {
return func() {
mut.Lock()
defer mut.Unlock()
if debug {
log.Println("Evicting", relay)
}
for i, current := range knownRelays {
if current.uri.Host == relay.uri.Host {
if debug {
log.Println("Evicted", relay)
}
last := len(knownRelays) - 1
knownRelays[i] = knownRelays[last]
knownRelays = knownRelays[:last]
deleteMetrics(current.uri.Host)
}
}
delete(evictionTimers, relay.uri.Host)
}
}
func limit(addr string, cache *lru.Cache, lock sync.RWMutex, intv time.Duration, burst int) bool {
if host, _, err := net.SplitHostPort(addr); err == nil {
addr = host
}
lock.RLock()
bkt, ok := cache.Get(addr)
lock.RUnlock()
if ok {
bkt := bkt.(*rate.Limiter)
if !bkt.Allow() {
// Rate limit
return true
}
} else {
lock.Lock()
cache.Add(addr, rate.NewLimiter(rate.Every(intv), burst))
lock.Unlock()
}
return false
}
func loadRelays(file string) []*relay {
content, err := ioutil.ReadFile(file)
if err != nil {
log.Println("Failed to load relays: " + err.Error())
return nil
}
var relays []*relay
for _, line := range strings.Split(string(content), "\n") {
if len(line) == 0 {
continue
}
uri, err := url.Parse(line)
if err != nil {
if debug {
log.Println("Skipping relay", line, "due to parse error", err)
}
continue
}
relays = append(relays, &relay{
URL: line,
Location: getLocation(uri.Host),
uri: uri,
})
if debug {
log.Println("Adding relay", line)
}
}
return relays
}
func saveRelays(file string, relays []*relay) error {
var content string
for _, relay := range relays {
content += relay.uri.String() + "\n"
}
return ioutil.WriteFile(file, []byte(content), 0777)
}
func createTestCertificate() tls.Certificate {
tmpDir, err := ioutil.TempDir("", "relaypoolsrv")
if err != nil {
log.Fatal(err)
}
certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv")
if err != nil {
log.Fatalln("Failed to create test X509 key pair:", err)
}
return cert
}
func getLocation(host string) location {
timer := prometheus.NewTimer(locationLookupSeconds)
defer timer.ObserveDuration()
db, err := geoip2.Open(geoipPath)
if err != nil {
return location{}
}
defer db.Close()
addr, err := net.ResolveTCPAddr("tcp", host)
if err != nil {
return location{}
}
city, err := db.City(addr.IP)
if err != nil {
return location{}
}
return location{
Longitude: city.Location.Longitude,
Latitude: city.Location.Latitude,
City: city.City.Names["en"],
Country: city.Country.IsoCode,
Continent: city.Continent.Code,
}
}
type loggingResponseWriter struct {
http.ResponseWriter
statusCode int
}
func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
return &loggingResponseWriter{w, http.StatusOK}
}
func (lrw *loggingResponseWriter) WriteHeader(code int) {
lrw.statusCode = code
lrw.ResponseWriter.WriteHeader(code)
}

View File

@@ -1,259 +0,0 @@
// Copyright (C) 2018 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
package main
import (
"encoding/json"
"net"
"net/http"
"os"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/syncthing/syncthing/lib/sync"
)
func init() {
processCollectorOpts := prometheus.ProcessCollectorOpts{
Namespace: "syncthing_relaypoolsrv",
PidFn: func() (int, error) {
return os.Getpid(), nil
},
}
prometheus.MustRegister(
prometheus.NewProcessCollector(processCollectorOpts),
)
}
var (
statusClient = http.Client{
Timeout: 5 * time.Second,
}
apiRequestsTotal = makeCounter("api_requests_total", "Number of API requests.", "type", "result")
apiRequestsSeconds = makeSummary("api_requests_seconds", "Latency of API requests.", "type")
relayTestsTotal = makeCounter("tests_total", "Number of relay tests.", "result")
relayTestActionsSeconds = makeSummary("test_actions_seconds", "Latency of relay test actions.", "type")
locationLookupSeconds = makeSummary("location_lookup_seconds", "Latency of location lookups.").WithLabelValues()
metricsRequestsSeconds = makeSummary("metrics_requests_seconds", "Latency of metric requests.").WithLabelValues()
scrapeSeconds = makeSummary("relay_scrape_seconds", "Latency of metric scrapes from remote relays.", "result")
relayUptime = makeGauge("relay_uptime", "Uptime of relay", "relay")
relayPendingSessionKeys = makeGauge("relay_pending_session_keys", "Number of pending session keys (two keys per session, one per each side of the connection)", "relay")
relayActiveSessions = makeGauge("relay_active_sessions", "Number of sessions that are happening, a session contains two parties", "relay")
relayConnections = makeGauge("relay_connections", "Number of devices connected to the relay", "relay")
relayProxies = makeGauge("relay_proxies", "Number of active proxy routines sending data between peers (two proxies per session, one for each way)", "relay")
relayBytesProxied = makeGauge("relay_bytes_proxied", "Number of bytes proxied by the relay", "relay")
relayGoRoutines = makeGauge("relay_go_routines", "Number of Go routines in the process", "relay")
relaySessionRate = makeGauge("relay_session_rate", "Rate applied per session", "relay")
relayGlobalRate = makeGauge("relay_global_rate", "Global rate applied on the whole relay", "relay")
relayBuildInfo = makeGauge("relay_build_info", "Build information about a relay", "relay", "go_version", "go_os", "go_arch")
relayLocationInfo = makeGauge("relay_location_info", "Location information about a relay", "relay", "city", "country", "continent")
lastStats = make(map[string]stats)
)
func makeGauge(name string, help string, labels ...string) *prometheus.GaugeVec {
gauge := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: "syncthing",
Subsystem: "relaypoolsrv",
Name: name,
Help: help,
},
labels,
)
prometheus.MustRegister(gauge)
return gauge
}
func makeSummary(name string, help string, labels ...string) *prometheus.SummaryVec {
summary := prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "syncthing",
Subsystem: "relaypoolsrv",
Name: name,
Help: help,
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
labels,
)
prometheus.MustRegister(summary)
return summary
}
func makeCounter(name string, help string, labels ...string) *prometheus.CounterVec {
counter := prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "relaypoolsrv",
Name: name,
Help: help,
},
labels,
)
prometheus.MustRegister(counter)
return counter
}
func statsRefresher(interval time.Duration) {
ticker := time.NewTicker(interval)
for range ticker.C {
refreshStats()
}
}
type statsFetchResult struct {
relay *relay
stats *stats
}
func refreshStats() {
mut.RLock()
relays := append(permanentRelays, knownRelays...)
mut.RUnlock()
now := time.Now()
wg := sync.NewWaitGroup()
results := make(chan statsFetchResult, len(relays))
for _, rel := range relays {
wg.Add(1)
go func(rel *relay) {
t0 := time.Now()
stats := fetchStats(rel)
duration := time.Since(t0).Seconds()
result := "success"
if stats == nil {
result = "failed"
}
scrapeSeconds.WithLabelValues(result).Observe(duration)
results <- statsFetchResult{
relay: rel,
stats: fetchStats(rel),
}
wg.Done()
}(rel)
}
wg.Wait()
close(results)
mut.Lock()
relayBuildInfo.Reset()
relayLocationInfo.Reset()
for result := range results {
result.relay.StatsRetrieved = now
result.relay.Stats = result.stats
if result.stats == nil {
deleteMetrics(result.relay.uri.Host)
} else {
updateMetrics(result.relay.uri.Host, *result.stats, result.relay.Location)
}
}
mut.Unlock()
}
func fetchStats(relay *relay) *stats {
statusAddr := relay.uri.Query().Get("statusAddr")
if statusAddr == "" {
statusAddr = ":22070"
}
statusHost, statusPort, err := net.SplitHostPort(statusAddr)
if err != nil {
return nil
}
if statusHost == "" {
if host, _, err := net.SplitHostPort(relay.uri.Host); err != nil {
return nil
} else {
statusHost = host
}
}
url := "http://" + net.JoinHostPort(statusHost, statusPort) + "/status"
response, err := statusClient.Get(url)
if err != nil {
return nil
}
var stats stats
if json.NewDecoder(response.Body).Decode(&stats); err != nil {
return nil
}
return &stats
}
func updateMetrics(host string, stats stats, location location) {
if stats.GoVersion != "" || stats.GoOS != "" || stats.GoArch != "" {
relayBuildInfo.WithLabelValues(host, stats.GoVersion, stats.GoOS, stats.GoArch).Add(1)
}
if location.City != "" || location.Country != "" || location.Continent != "" {
relayLocationInfo.WithLabelValues(host, location.City, location.Country, location.Continent).Add(1)
}
if lastStat, ok := lastStats[host]; ok {
stats = mergeStats(stats, lastStat)
}
relayUptime.WithLabelValues(host).Set(float64(stats.UptimeSeconds))
relayPendingSessionKeys.WithLabelValues(host).Set(float64(stats.PendingSessionKeys))
relayActiveSessions.WithLabelValues(host).Set(float64(stats.ActiveSessions))
relayConnections.WithLabelValues(host).Set(float64(stats.Connections))
relayProxies.WithLabelValues(host).Set(float64(stats.Proxies))
relayBytesProxied.WithLabelValues(host).Set(float64(stats.BytesProxied))
relayGoRoutines.WithLabelValues(host).Set(float64(stats.GoRoutines))
relaySessionRate.WithLabelValues(host).Set(float64(stats.Options.SessionRate))
relayGlobalRate.WithLabelValues(host).Set(float64(stats.Options.GlobalRate))
lastStats[host] = stats
}
func deleteMetrics(host string) {
relayUptime.DeleteLabelValues(host)
relayPendingSessionKeys.DeleteLabelValues(host)
relayActiveSessions.DeleteLabelValues(host)
relayConnections.DeleteLabelValues(host)
relayProxies.DeleteLabelValues(host)
relayBytesProxied.DeleteLabelValues(host)
relayGoRoutines.DeleteLabelValues(host)
relaySessionRate.DeleteLabelValues(host)
relayGlobalRate.DeleteLabelValues(host)
delete(lastStats, host)
}
// Due to some unexplainable behaviour, some of the numbers sometimes travel slightly backwards (by less than 1%)
// This happens between scrapes, which is 30s, so this can't be a race.
// This causes prometheus to assume a "rate reset", hence causes phenomenal spikes.
// One of the number that moves backwards is BytesProxied, which atomically increments a counter with numeric value
// returned by net.Conn.Read(). I don't think that can return a negative value, so I have no idea what's going on.
func mergeStats(new stats, old stats) stats {
new.UptimeSeconds = mergeValue(new.UptimeSeconds, old.UptimeSeconds)
new.PendingSessionKeys = mergeValue(new.PendingSessionKeys, old.PendingSessionKeys)
new.ActiveSessions = mergeValue(new.ActiveSessions, old.ActiveSessions)
new.Connections = mergeValue(new.Connections, old.Connections)
new.Proxies = mergeValue(new.Proxies, old.Proxies)
new.BytesProxied = mergeValue(new.BytesProxied, old.BytesProxied)
new.GoRoutines = mergeValue(new.GoRoutines, old.GoRoutines)
new.Options.SessionRate = mergeValue(new.Options.SessionRate, old.Options.SessionRate)
new.Options.GlobalRate = mergeValue(new.Options.GlobalRate, old.Options.GlobalRate)
return new
}
func mergeValue(new, old int) int {
if new >= old {
return new // normal increase
}
if float64(new) > 0.99*float64(old) {
return old // slight backward movement
}
return new // reset (relay restart)
}

View File

@@ -1,21 +0,0 @@
// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
package main
import (
"testing"
)
func TestMerge(t *testing.T) {
if mergeValue(1001, 1000) != 1001 {
t.Error("the computer says no")
}
if mergeValue(999, 1000) != 1000 {
t.Error("the computer says no")
}
if mergeValue(1, 1000) != 1 {
t.Error("the computer says no")
}
}

View File

@@ -1,22 +0,0 @@
The MIT License (MIT)
Copyright (c) 2015 The Syncthing Project
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -1,131 +0,0 @@
strelaysrv
==========
This is the relay server for the `syncthing` project.
:exclamation:Warnings:exclamation: - Read or regret
-----
By default, all relay servers will join to the default public relay pool, which means that the relay server will be available for public use, and **will consume your bandwidth** helping others to connect.
If you wish to disable this behaviour, please specify the `-pools=""` argument.
Please note that `strelaysrv` is only usable by `syncthing` **version v0.12 and onwards**.
To run `strelaysrv` you need to have port 22067 available to the internet, which means you might need to port forward it and/or allow it through your firewall.
Furthermore, by default `strelaysrv` will also expose a /status HTTP endpoint on port 22070, which is used by the pool servers to read metrics of the `strelaysrv`, such as the current transfer rates, how many clients are connected, etc. If you wish this information to be available you may need to port forward and allow it through your firewall. This is not mandatory for the `strelaysrv` to function, and is used only to gather metrics and present them in the overview page of the pool server.
At the point of writing the endpoint output looks as follows:
```
{
"bytesProxied": 0,
"goArch": "amd64",
"goMaxProcs": 1,
"goNumRoutine": 13,
"goOS": "linux",
"goVersion": "go1.6",
"kbps10s1m5m15m30m60m": [
0,
0,
0,
0,
0,
0
],
"numActiveSessions": 0,
"numConnections": 0,
"numPendingSessionKeys": 2,
"numProxies": 0,
"options": {
"global-rate": 0,
"message-timeout": 60,
"network-timeout": 120,
"per-session-rate": 0,
"ping-interval": 60,
"pools": [
"https://relays.syncthing.net/endpoint"
],
"provided-by": ""
},
"startTime": "2016-03-06T12:53:07.090847749-05:00",
"uptimeSeconds": 17
}
```
If you wish to disable the /status endpoint, provide `-status-srv=""` as one of the arguments when starting the strelaysrv.
Running for public use
----
Make sure you have a public IP with port 22067 open, or have forwarded port 22067 if you are behind a NAT.
Run the `strelaysrv` with no arguments (or `-debug` if you want more output), and that should be enough for the server to join the public relay pool.
You should see a message saying:
```
2015/09/21 22:45:46 pool.go:60: Joined https://relays.syncthing.net/endpoint rejoining in 48m0s
```
See `strelaysrv -help` for other options, such as rate limits, timeout intervals, etc.
Running for private use
-----
Once you've started the `strelaysrv`, it will generate a key pair and print a URI:
```bash
relay://:22067/?id=EZQOIDM-6DDD4ZI-DJ65NSM-4OQWRAT-EIKSMJO-OZ552BO-WQZEGYY-STS5RQM&pingInterval=1m0s&networkTimeout=2m0s&sessionLimitBps=0&globalLimitBps=0&statusAddr=:22070
```
This URI contains a partial address of the relay server, as well as its options which in the future may be taken into account when choosing the most suitable relay.
Because the `-listen` option was not used `strelaysrv` does not know its external IP, therefore you should replace the host part of the URI with your public IP address on which the `strelaysrv` will be available:
```bash
relay://192.0.2.1:22067/?id=EZQOIDM-6DDD4ZI-DJ65NSM-4OQWRAT-EIKSMJO-OZ552BO-WQZEGYY-STS5RQM&pingInterval=1m0s&networkTimeout=2m0s&sessionLimitBps=0&globalLimitBps=0&statusAddr=:22070
```
If you do not care about certificate pinning (improved security) or do not care about passing verbose settings to the clients, you can shorten the URL to just the host part:
```bash
relay://192.0.2.1:22067
```
This URI can then be used in `syncthing` clients as one of the relay servers by adding the URI to the "Sync Protocol Listen Address" field, under Actions and Settings.
See `strelaysrv -help` for other options, such as rate limits, timeout intervals, etc.
Other items available in this repo
----
##### testutil
A test utility which can be used to test the connectivity of a relay server.
You need to generate two x509 key pairs (key.pem and cert.pem), one for the client and one for the server, in separate directories.
Afterwards, start the client:
```bash
./testutil -relay="relay://192.0.2.1:22067" -keys=certs/client/ -join
```
This prints out the client ID:
```
2015/09/21 23:00:52 main.go:42: ID: BG2C5ZA-W7XPFDO-LH222Z6-65F3HJX-ADFTGRT-3SBFIGM-KV26O2Q-E5RMRQ2
```
In the other terminal run the following:
```bash
./testutil -relay="relay://192.0.2.1:22067" -keys=certs/server/ -connect=BG2C5ZA-W7XPFDO-LH222Z6-65F3HJX-ADFTGRT-3SBFIGM-KV26O2Q-E5RMRQ2
```
Which should then give you an interactive prompt, where you can type things in one terminal, and they get relayed to the other terminal.
Relay related libraries used by this repo
----
##### Relay protocol definition.
[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/protocol)
##### Relay client
Only used by the testutil.
[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/client)

View File

@@ -1,9 +0,0 @@
[strelaysrv]
title=Syncthing relay server
description=Proxies traffic of syncthing client behind firewalls
ports=22067/tcp
[strelaysrv-metrics]
title=Syncthing relay metrics
description=Provides metrics about the syncthing relay server
ports=22070/tcp

View File

@@ -1,5 +0,0 @@
# Default settings for syncthing-relaysrv (strelaysrv).
NAT=true
## Add Options here:
RELAYSRV_OPTS=

View File

@@ -1,25 +0,0 @@
[Unit]
Description=Syncthing Relay Server
After=network.target
Documentation=man:strelaysrv(1)
[Service]
WorkingDirectory=/var/lib/syncthing-relaysrv
EnvironmentFile=/etc/default/syncthing-relaysrv
ExecStart=/usr/bin/strelaysrv -nat=${NAT} $RELAYSRV_OPTS
# Hardening
User=syncthing-relaysrv
Group=syncthing
ProtectSystem=strict
ReadWritePaths=/var/lib/syncthing-relaysrv
NoNewPrivileges=true
PrivateTmp=true
PrivateDevices=true
ProtectHome=true
SystemCallArchitectures=native
MemoryDenyWriteExecute=true
[Install]
WantedBy=multi-user.target
Alias=syncthing-relaysrv.service

View File

@@ -1,357 +0,0 @@
// Copyright (C) 2015 Audrius Butkevicius and Contributors.
package main
import (
"crypto/tls"
"encoding/hex"
"log"
"net"
"sync"
"sync/atomic"
"time"
syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/tlsutil"
"github.com/syncthing/syncthing/lib/relay/protocol"
)
var (
outboxesMut = sync.RWMutex{}
outboxes = make(map[syncthingprotocol.DeviceID]chan interface{})
numConnections int64
)
func listener(proto, addr string, config *tls.Config) {
tcpListener, err := net.Listen("tcp", addr)
if err != nil {
log.Fatalln(err)
}
listener := tlsutil.DowngradingListener{
Listener: tcpListener,
}
for {
conn, isTLS, err := listener.AcceptNoWrapTLS()
if err != nil {
if debug {
log.Println("Listener failed to accept connection from", conn.RemoteAddr(), ". Possibly a TCP Ping.")
}
continue
}
setTCPOptions(conn)
if debug {
log.Println("Listener accepted connection from", conn.RemoteAddr(), "tls", isTLS)
}
if isTLS {
go protocolConnectionHandler(conn, config)
} else {
go sessionConnectionHandler(conn)
}
}
}
func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
conn := tls.Server(tcpConn, config)
if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil {
if debug {
log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr())
}
conn.Close()
return
}
err := conn.Handshake()
if err != nil {
if debug {
log.Println("Protocol connection TLS handshake:", conn.RemoteAddr(), err)
}
conn.Close()
return
}
state := conn.ConnectionState()
if (!state.NegotiatedProtocolIsMutual || state.NegotiatedProtocol != protocol.ProtocolName) && debug {
log.Println("Protocol negotiation error")
}
certs := state.PeerCertificates
if len(certs) != 1 {
if debug {
log.Println("Certificate list error")
}
conn.Close()
return
}
conn.SetDeadline(time.Time{})
id := syncthingprotocol.NewDeviceID(certs[0].Raw)
messages := make(chan interface{})
errors := make(chan error, 1)
outbox := make(chan interface{})
// Read messages from the connection and send them on the messages
// channel. When there is an error, send it on the error channel and
// return. Applies also when the connection gets closed, so the pattern
// below is to close the connection on error, then wait for the error
// signal from messageReader to exit.
go messageReader(conn, messages, errors)
pingTicker := time.NewTicker(pingInterval)
defer pingTicker.Stop()
timeoutTicker := time.NewTimer(networkTimeout)
defer timeoutTicker.Stop()
joined := false
for {
select {
case message := <-messages:
timeoutTicker.Reset(networkTimeout)
if debug {
log.Printf("Message %T from %s", message, id)
}
switch msg := message.(type) {
case protocol.JoinRelayRequest:
if atomic.LoadInt32(&overLimit) > 0 {
protocol.WriteMessage(conn, protocol.RelayFull{})
if debug {
log.Println("Refusing join request from", id, "due to being over limits")
}
conn.Close()
limitCheckTimer.Reset(time.Second)
continue
}
outboxesMut.RLock()
_, ok := outboxes[id]
outboxesMut.RUnlock()
if ok {
protocol.WriteMessage(conn, protocol.ResponseAlreadyConnected)
if debug {
log.Println("Already have a peer with the same ID", id, conn.RemoteAddr())
}
conn.Close()
continue
}
outboxesMut.Lock()
outboxes[id] = outbox
outboxesMut.Unlock()
joined = true
protocol.WriteMessage(conn, protocol.ResponseSuccess)
case protocol.ConnectRequest:
requestedPeer := syncthingprotocol.DeviceIDFromBytes(msg.ID)
outboxesMut.RLock()
peerOutbox, ok := outboxes[requestedPeer]
outboxesMut.RUnlock()
if !ok {
if debug {
log.Println(id, "is looking for", requestedPeer, "which does not exist")
}
protocol.WriteMessage(conn, protocol.ResponseNotFound)
conn.Close()
continue
}
// requestedPeer is the server, id is the client
ses := newSession(requestedPeer, id, sessionLimiter, globalLimiter)
go ses.Serve()
clientInvitation := ses.GetClientInvitationMessage()
serverInvitation := ses.GetServerInvitationMessage()
if err := protocol.WriteMessage(conn, clientInvitation); err != nil {
if debug {
log.Printf("Error sending invitation from %s to client: %s", id, err)
}
conn.Close()
continue
}
select {
case peerOutbox <- serverInvitation:
if debug {
log.Println("Sent invitation from", id, "to", requestedPeer)
}
case <-time.After(time.Second):
if debug {
log.Println("Could not send invitation from", id, "to", requestedPeer, "as peer disconnected")
}
}
conn.Close()
case protocol.Ping:
if err := protocol.WriteMessage(conn, protocol.Pong{}); err != nil {
if debug {
log.Println("Error writing pong:", err)
}
conn.Close()
continue
}
case protocol.Pong:
// Nothing
default:
if debug {
log.Printf("Unknown message %s: %T", id, message)
}
protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage)
conn.Close()
}
case err := <-errors:
if debug {
log.Printf("Closing connection %s: %s", id, err)
}
// Potentially closing a second time.
conn.Close()
if joined {
// Only delete the outbox if the client is joined, as it might be
// a lookup request coming from the same client.
outboxesMut.Lock()
delete(outboxes, id)
outboxesMut.Unlock()
// Also, kill all sessions related to this node, as it probably
// went offline. This is for the other end to realize the client
// is no longer there faster. This also helps resolve
// 'already connected' errors when one of the sides is
// restarting, and connecting to the other peer before the other
// peer even realised that the node has gone away.
dropSessions(id)
}
return
case <-pingTicker.C:
if !joined {
if debug {
log.Println(id, "didn't join within", pingInterval)
}
conn.Close()
continue
}
if err := protocol.WriteMessage(conn, protocol.Ping{}); err != nil {
if debug {
log.Println(id, err)
}
conn.Close()
}
if atomic.LoadInt32(&overLimit) > 0 && !hasSessions(id) {
if debug {
log.Println("Dropping", id, "as it has no sessions and we are over our limits")
}
protocol.WriteMessage(conn, protocol.RelayFull{})
conn.Close()
limitCheckTimer.Reset(time.Second)
}
case <-timeoutTicker.C:
// We should receive a error from the reader loop, which will cause
// us to quit this loop.
if debug {
log.Printf("%s timed out", id)
}
conn.Close()
case msg := <-outbox:
if debug {
log.Printf("Sending message %T to %s", msg, id)
}
if err := protocol.WriteMessage(conn, msg); err != nil {
if debug {
log.Println(id, err)
}
conn.Close()
}
}
}
}
func sessionConnectionHandler(conn net.Conn) {
if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil {
if debug {
log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr())
}
conn.Close()
return
}
message, err := protocol.ReadMessage(conn)
if err != nil {
return
}
switch msg := message.(type) {
case protocol.JoinSessionRequest:
ses := findSession(string(msg.Key))
if debug {
log.Println(conn.RemoteAddr(), "session lookup", ses, hex.EncodeToString(msg.Key)[:5])
}
if ses == nil {
protocol.WriteMessage(conn, protocol.ResponseNotFound)
conn.Close()
return
}
if !ses.AddConnection(conn) {
if debug {
log.Println("Failed to add", conn.RemoteAddr(), "to session", ses)
}
protocol.WriteMessage(conn, protocol.ResponseAlreadyConnected)
conn.Close()
return
}
if err := protocol.WriteMessage(conn, protocol.ResponseSuccess); err != nil {
if debug {
log.Println("Failed to send session join response to ", conn.RemoteAddr(), "for", ses)
}
return
}
if err := conn.SetDeadline(time.Time{}); err != nil {
if debug {
log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr())
}
conn.Close()
return
}
default:
if debug {
log.Println("Unexpected message from", conn.RemoteAddr(), message)
}
protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage)
conn.Close()
}
}
func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) {
atomic.AddInt64(&numConnections, 1)
defer atomic.AddInt64(&numConnections, -1)
for {
msg, err := protocol.ReadMessage(conn)
if err != nil {
errors <- err
return
}
messages <- msg
}
}

View File

@@ -1,312 +0,0 @@
// Copyright (C) 2015 Audrius Butkevicius and Contributors.
package main
import (
"crypto/tls"
"flag"
"fmt"
"log"
"net"
"net/http"
"net/url"
"os"
"os/signal"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync/atomic"
"syscall"
"time"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/relay/protocol"
"github.com/syncthing/syncthing/lib/tlsutil"
"golang.org/x/time/rate"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/nat"
_ "github.com/syncthing/syncthing/lib/pmp"
_ "github.com/syncthing/syncthing/lib/upnp"
syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
)
var (
Version string
BuildStamp string
BuildUser string
BuildHost string
BuildDate time.Time
LongVersion string
)
func init() {
stamp, _ := strconv.Atoi(BuildStamp)
BuildDate = time.Unix(int64(stamp), 0)
date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
LongVersion = fmt.Sprintf(`strelaysrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
}
var (
listen string
debug bool
sessionAddress []byte
sessionPort uint16
networkTimeout = 2 * time.Minute
pingInterval = time.Minute
messageTimeout = time.Minute
limitCheckTimer *time.Timer
sessionLimitBps int
globalLimitBps int
overLimit int32
descriptorLimit int64
sessionLimiter *rate.Limiter
globalLimiter *rate.Limiter
networkBufferSize int
statusAddr string
poolAddrs string
pools []string
providedBy string
defaultPoolAddrs = "https://relays.syncthing.net/endpoint"
natEnabled bool
natLease int
natRenewal int
natTimeout int
pprofEnabled bool
)
// httpClient is the HTTP client we use for outbound requests. It has a
// timeout and may get further options set during initialization.
var httpClient = &http.Client{
Timeout: 30 * time.Second,
}
func main() {
log.SetFlags(log.Lshortfile | log.LstdFlags)
var dir, extAddress, proto string
flag.StringVar(&listen, "listen", ":22067", "Protocol listen address")
flag.StringVar(&dir, "keys", ".", "Directory where cert.pem and key.pem is stored")
flag.DurationVar(&networkTimeout, "network-timeout", networkTimeout, "Timeout for network operations between the client and the relay.\n\tIf no data is received between the client and the relay in this period of time, the connection is terminated.\n\tFurthermore, if no data is sent between either clients being relayed within this period of time, the session is also terminated.")
flag.DurationVar(&pingInterval, "ping-interval", pingInterval, "How often pings are sent")
flag.DurationVar(&messageTimeout, "message-timeout", messageTimeout, "Maximum amount of time we wait for relevant messages to arrive")
flag.IntVar(&sessionLimitBps, "per-session-rate", sessionLimitBps, "Per session rate limit, in bytes/s")
flag.IntVar(&globalLimitBps, "global-rate", globalLimitBps, "Global rate limit, in bytes/s")
flag.BoolVar(&debug, "debug", debug, "Enable debug output")
flag.StringVar(&statusAddr, "status-srv", ":22070", "Listen address for status service (blank to disable)")
flag.StringVar(&poolAddrs, "pools", defaultPoolAddrs, "Comma separated list of relay pool addresses to join")
flag.StringVar(&providedBy, "provided-by", "", "An optional description about who provides the relay")
flag.StringVar(&extAddress, "ext-address", "", "An optional address to advertise as being available on.\n\tAllows listening on an unprivileged port with port forwarding from e.g. 443, and be connected to on port 443.")
flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
flag.BoolVar(&natEnabled, "nat", false, "Use UPnP/NAT-PMP to acquire external port mapping")
flag.IntVar(&natLease, "nat-lease", 60, "NAT lease length in minutes")
flag.IntVar(&natRenewal, "nat-renewal", 30, "NAT renewal frequency in minutes")
flag.IntVar(&natTimeout, "nat-timeout", 10, "NAT discovery timeout in seconds")
flag.BoolVar(&pprofEnabled, "pprof", false, "Enable the built in profiling on the status server")
flag.IntVar(&networkBufferSize, "network-buffer", 2048, "Network buffer size (two of these per proxied connection)")
flag.Parse()
if extAddress == "" {
extAddress = listen
}
if len(providedBy) > 30 {
log.Fatal("Provided-by cannot be longer than 30 characters")
}
addr, err := net.ResolveTCPAddr(proto, extAddress)
if err != nil {
log.Fatal(err)
}
laddr, err := net.ResolveTCPAddr(proto, listen)
if err != nil {
log.Fatal(err)
}
if laddr.IP != nil && !laddr.IP.IsUnspecified() {
// We bind to a specific address. Our outgoing HTTP requests should
// also come from that address.
laddr.Port = 0
boundDialer := &net.Dialer{LocalAddr: laddr}
httpClient.Transport = &http.Transport{
DialContext: boundDialer.DialContext,
}
}
log.Println(LongVersion)
maxDescriptors, err := osutil.MaximizeOpenFileLimit()
if maxDescriptors > 0 {
// Assume that 20% of FD's are leaked/unaccounted for.
descriptorLimit = int64(maxDescriptors*80) / 100
log.Println("Connection limit", descriptorLimit)
go monitorLimits()
} else if err != nil && runtime.GOOS != "windows" {
log.Println("Assuming no connection limit, due to error retrieving rlimits:", err)
}
sessionAddress = addr.IP[:]
sessionPort = uint16(addr.Port)
certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem")
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
log.Println("Failed to load keypair. Generating one, this might take a while...")
cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv")
if err != nil {
log.Fatalln("Failed to generate X509 key pair:", err)
}
}
tlsCfg := &tls.Config{
Certificates: []tls.Certificate{cert},
NextProtos: []string{protocol.ProtocolName},
ClientAuth: tls.RequestClientCert,
SessionTicketsDisabled: true,
InsecureSkipVerify: true,
MinVersion: tls.VersionTLS12,
CipherSuites: []uint16{
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
},
}
id := syncthingprotocol.NewDeviceID(cert.Certificate[0])
if debug {
log.Println("ID:", id)
}
wrapper := config.Wrap("config", config.New(id))
wrapper.SetOptions(config.OptionsConfiguration{
NATLeaseM: natLease,
NATRenewalM: natRenewal,
NATTimeoutS: natTimeout,
})
natSvc := nat.NewService(id, wrapper)
mapping := mapping{natSvc.NewMapping(nat.TCP, addr.IP, addr.Port)}
if natEnabled {
go natSvc.Serve()
found := make(chan struct{})
mapping.OnChanged(func(_ *nat.Mapping, _, _ []nat.Address) {
select {
case found <- struct{}{}:
default:
}
})
// Need to wait a few extra seconds, since NAT library waits exactly natTimeout seconds on all interfaces.
timeout := time.Duration(natTimeout+2) * time.Second
log.Printf("Waiting %s to acquire NAT mapping", timeout)
select {
case <-found:
log.Printf("Found NAT mapping: %s", mapping.ExternalAddresses())
case <-time.After(timeout):
log.Println("Timeout out waiting for NAT mapping.")
}
}
if sessionLimitBps > 0 {
sessionLimiter = rate.NewLimiter(rate.Limit(sessionLimitBps), 2*sessionLimitBps)
}
if globalLimitBps > 0 {
globalLimiter = rate.NewLimiter(rate.Limit(globalLimitBps), 2*globalLimitBps)
}
if statusAddr != "" {
go statusService(statusAddr)
}
uri, err := url.Parse(fmt.Sprintf("relay://%s/?id=%s&pingInterval=%s&networkTimeout=%s&sessionLimitBps=%d&globalLimitBps=%d&statusAddr=%s&providedBy=%s", mapping.Address(), id, pingInterval, networkTimeout, sessionLimitBps, globalLimitBps, statusAddr, providedBy))
if err != nil {
log.Fatalln("Failed to construct URI", err)
}
log.Println("URI:", uri.String())
if poolAddrs == defaultPoolAddrs {
log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
log.Println("!! Joining default relay pools, this relay will be available for public use. !!")
log.Println(`!! Use the -pools="" command line option to make the relay private. !!`)
log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
}
pools = strings.Split(poolAddrs, ",")
for _, pool := range pools {
pool = strings.TrimSpace(pool)
if len(pool) > 0 {
go poolHandler(pool, uri, mapping)
}
}
go listener(proto, listen, tlsCfg)
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
<-sigs
// Gracefully close all connections, hoping that clients will be faster
// to realize that the relay is now gone.
sessionMut.RLock()
for _, session := range activeSessions {
session.CloseConns()
}
for _, session := range pendingSessions {
session.CloseConns()
}
sessionMut.RUnlock()
outboxesMut.RLock()
for _, outbox := range outboxes {
close(outbox)
}
outboxesMut.RUnlock()
time.Sleep(500 * time.Millisecond)
}
func monitorLimits() {
limitCheckTimer = time.NewTimer(time.Minute)
for range limitCheckTimer.C {
if atomic.LoadInt64(&numConnections)+atomic.LoadInt64(&numProxies) > descriptorLimit {
atomic.StoreInt32(&overLimit, 1)
log.Println("Gone past our connection limits. Starting to refuse new/drop idle connections.")
} else if atomic.CompareAndSwapInt32(&overLimit, 1, 0) {
log.Println("Dropped below our connection limits. Accepting new connections.")
}
limitCheckTimer.Reset(time.Minute)
}
}
type mapping struct {
*nat.Mapping
}
func (m *mapping) Address() nat.Address {
ext := m.ExternalAddresses()
if len(ext) > 0 {
return ext[0]
}
return m.Mapping.Address()
}

View File

@@ -1,65 +0,0 @@
// Copyright (C) 2015 Audrius Butkevicius and Contributors.
package main
import (
"bytes"
"encoding/json"
"io/ioutil"
"log"
"net/url"
"time"
)
func poolHandler(pool string, uri *url.URL, mapping mapping) {
if debug {
log.Println("Joining", pool)
}
for {
uriCopy := *uri
uriCopy.Host = mapping.Address().String()
var b bytes.Buffer
json.NewEncoder(&b).Encode(struct {
URL string `json:"url"`
}{
uriCopy.String(),
})
resp, err := httpClient.Post(pool, "application/json", &b)
if err != nil {
log.Println("Error joining pool", pool, err)
} else if resp.StatusCode == 500 {
bs, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println("Failed to join", pool, "due to an internal server error. Could not read response:", err)
} else {
log.Println("Failed to join", pool, "due to an internal server error:", string(bs))
}
resp.Body.Close()
} else if resp.StatusCode == 429 {
log.Println(pool, "under load, will retry in a minute")
time.Sleep(time.Minute)
continue
} else if resp.StatusCode == 401 {
log.Println(pool, "failed to join due to IP address not matching external address. Aborting")
return
} else if resp.StatusCode == 200 {
var x struct {
EvictionIn time.Duration `json:"evictionIn"`
}
err := json.NewDecoder(resp.Body).Decode(&x)
if err == nil {
rejoin := x.EvictionIn - (x.EvictionIn / 5)
log.Println("Joined", pool, "rejoining in", rejoin)
time.Sleep(rejoin)
continue
} else {
log.Println("Failed to deserialize response", err)
}
} else {
log.Println(pool, "unknown response type from server", resp.StatusCode)
}
time.Sleep(time.Hour)
}
}

View File

@@ -1,4 +0,0 @@
#!/bin/bash
addgroup --system syncthing
adduser --system --home /var/lib/syncthing-relaysrv --ingroup syncthing syncthing-relaysrv

View File

@@ -1,353 +0,0 @@
// Copyright (C) 2015 Audrius Butkevicius and Contributors.
package main
import (
"crypto/rand"
"encoding/hex"
"fmt"
"log"
"math"
"net"
"sync"
"sync/atomic"
"time"
"golang.org/x/time/rate"
syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/relay/protocol"
)
var (
sessionMut = sync.RWMutex{}
activeSessions = make([]*session, 0)
pendingSessions = make(map[string]*session)
numProxies int64
bytesProxied int64
)
func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionRateLimit, globalRateLimit *rate.Limiter) *session {
serverkey := make([]byte, 32)
_, err := rand.Read(serverkey)
if err != nil {
return nil
}
clientkey := make([]byte, 32)
_, err = rand.Read(clientkey)
if err != nil {
return nil
}
ses := &session{
serverkey: serverkey,
serverid: serverid,
clientkey: clientkey,
clientid: clientid,
rateLimit: makeRateLimitFunc(sessionRateLimit, globalRateLimit),
connsChan: make(chan net.Conn),
conns: make([]net.Conn, 0, 2),
}
if debug {
log.Println("New session", ses)
}
sessionMut.Lock()
pendingSessions[string(ses.serverkey)] = ses
pendingSessions[string(ses.clientkey)] = ses
sessionMut.Unlock()
return ses
}
func findSession(key string) *session {
sessionMut.Lock()
defer sessionMut.Unlock()
ses, ok := pendingSessions[key]
if !ok {
return nil
}
delete(pendingSessions, key)
return ses
}
func dropSessions(id syncthingprotocol.DeviceID) {
sessionMut.RLock()
for _, session := range activeSessions {
if session.HasParticipant(id) {
if debug {
log.Println("Dropping session", session, "involving", id)
}
session.CloseConns()
}
}
sessionMut.RUnlock()
}
func hasSessions(id syncthingprotocol.DeviceID) bool {
sessionMut.RLock()
has := false
for _, session := range activeSessions {
if session.HasParticipant(id) {
has = true
break
}
}
sessionMut.RUnlock()
return has
}
type session struct {
mut sync.Mutex
serverkey []byte
serverid syncthingprotocol.DeviceID
clientkey []byte
clientid syncthingprotocol.DeviceID
rateLimit func(bytes int)
connsChan chan net.Conn
conns []net.Conn
}
func (s *session) AddConnection(conn net.Conn) bool {
if debug {
log.Println("New connection for", s, "from", conn.RemoteAddr())
}
select {
case s.connsChan <- conn:
return true
default:
}
return false
}
func (s *session) Serve() {
timedout := time.After(messageTimeout)
if debug {
log.Println("Session", s, "serving")
}
for {
select {
case conn := <-s.connsChan:
s.mut.Lock()
s.conns = append(s.conns, conn)
s.mut.Unlock()
// We're the only ones mutating s.conns, hence we are free to read it.
if len(s.conns) < 2 {
continue
}
close(s.connsChan)
if debug {
log.Println("Session", s, "starting between", s.conns[0].RemoteAddr(), "and", s.conns[1].RemoteAddr())
}
wg := sync.WaitGroup{}
wg.Add(2)
var err0 error
go func() {
err0 = s.proxy(s.conns[0], s.conns[1])
wg.Done()
}()
var err1 error
go func() {
err1 = s.proxy(s.conns[1], s.conns[0])
wg.Done()
}()
sessionMut.Lock()
activeSessions = append(activeSessions, s)
sessionMut.Unlock()
wg.Wait()
if debug {
log.Println("Session", s, "ended, outcomes:", err0, "and", err1)
}
goto done
case <-timedout:
if debug {
log.Println("Session", s, "timed out")
}
goto done
}
}
done:
// We can end up here in 3 cases:
// 1. Timeout joining, in which case there are potentially entries in pendingSessions
// 2. General session end/timeout, in which case there are entries in activeSessions
// 3. Protocol handler calls dropSession as one of its clients disconnects.
sessionMut.Lock()
delete(pendingSessions, string(s.serverkey))
delete(pendingSessions, string(s.clientkey))
for i, session := range activeSessions {
if session == s {
l := len(activeSessions) - 1
activeSessions[i] = activeSessions[l]
activeSessions[l] = nil
activeSessions = activeSessions[:l]
}
}
sessionMut.Unlock()
// If we are here because of case 2 or 3, we are potentially closing some or
// all connections a second time.
s.CloseConns()
if debug {
log.Println("Session", s, "stopping")
}
}
func (s *session) GetClientInvitationMessage() protocol.SessionInvitation {
return protocol.SessionInvitation{
From: s.serverid[:],
Key: s.clientkey,
Address: sessionAddress,
Port: sessionPort,
ServerSocket: false,
}
}
func (s *session) GetServerInvitationMessage() protocol.SessionInvitation {
return protocol.SessionInvitation{
From: s.clientid[:],
Key: s.serverkey,
Address: sessionAddress,
Port: sessionPort,
ServerSocket: true,
}
}
func (s *session) HasParticipant(id syncthingprotocol.DeviceID) bool {
return s.clientid == id || s.serverid == id
}
func (s *session) CloseConns() {
s.mut.Lock()
for _, conn := range s.conns {
conn.Close()
}
s.mut.Unlock()
}
func (s *session) proxy(c1, c2 net.Conn) error {
if debug {
log.Println("Proxy", c1.RemoteAddr(), "->", c2.RemoteAddr())
}
atomic.AddInt64(&numProxies, 1)
defer atomic.AddInt64(&numProxies, -1)
buf := make([]byte, networkBufferSize)
for {
c1.SetReadDeadline(time.Now().Add(networkTimeout))
n, err := c1.Read(buf)
if err != nil {
return err
}
atomic.AddInt64(&bytesProxied, int64(n))
if debug {
log.Printf("%d bytes from %s to %s", n, c1.RemoteAddr(), c2.RemoteAddr())
}
if s.rateLimit != nil {
s.rateLimit(n)
}
c2.SetWriteDeadline(time.Now().Add(networkTimeout))
_, err = c2.Write(buf[:n])
if err != nil {
return err
}
}
}
func (s *session) String() string {
return fmt.Sprintf("<%s/%s>", hex.EncodeToString(s.clientkey)[:5], hex.EncodeToString(s.serverkey)[:5])
}
func makeRateLimitFunc(sessionRateLimit, globalRateLimit *rate.Limiter) func(int) {
// This may be a case of super duper premature optimization... We build an
// optimized function to do the rate limiting here based on what we need
// to do and then use it in the loop.
if sessionRateLimit == nil && globalRateLimit == nil {
// No limiting needed. We could equally well return a func(int64){} and
// not do a nil check were we use it, but I think the nil check there
// makes it clear that there will be no limiting if none is
// configured...
return nil
}
if sessionRateLimit == nil {
// We only have a global limiter
return func(bytes int) {
take(bytes, globalRateLimit)
}
}
if globalRateLimit == nil {
// We only have a session limiter
return func(bytes int) {
take(bytes, sessionRateLimit)
}
}
// We have both. Queue the bytes on both the global and session specific
// rate limiters.
return func(bytes int) {
take(bytes, sessionRateLimit, globalRateLimit)
}
}
// take is a utility function to consume tokens from a set of rate.Limiters.
// Tokens are consumed in parallel on all limiters, respecting their
// individual burst sizes.
func take(tokens int, ls ...*rate.Limiter) {
// minBurst is the smallest burst size supported by all limiters.
minBurst := int(math.MaxInt32)
for _, l := range ls {
if burst := l.Burst(); burst < minBurst {
minBurst = burst
}
}
for tokens > 0 {
// chunk is how many tokens we can consume at a time
chunk := tokens
if chunk > minBurst {
chunk = minBurst
}
// maxDelay is the longest delay mandated by any of the limiters for
// the chosen chunk size.
var maxDelay time.Duration
for _, l := range ls {
res := l.ReserveN(time.Now(), chunk)
if del := res.Delay(); del > maxDelay {
maxDelay = del
}
}
time.Sleep(maxDelay)
tokens -= chunk
}
}

Some files were not shown because too many files have changed in this diff Show More